mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Add location to expressions. Change symboltable to use flags for symbols.

Browse Source
This commit is contained in:
Windel Bouwman committed 2019-07-20 20:44:38 +02:00
1 parent 1b48ade07b
commit d6c19c1c6a
10 files changed
+935 -828
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

compiler/src/compile.rs
+100 -96
View File
@@ -6,7 +6,7 @@
//! https://github.com/micropython/micropython/blob/master/py/compile.c
use crate::error::{CompileError, CompileErrorType};
use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope};
use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope};
use num_complex::Complex64;
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
use rustpython_parser::{ast, parser};
@@ -75,7 +75,7 @@ pub fn compile_program(
/// Compile a single Python expression to bytecode
pub fn compile_statement_eval(
statement: Vec<ast::LocatedStatement>,
statement: Vec<ast::Statement>,
source_path: String,
optimize: u8,
) -> Result<CodeObject, CompileError> {
@@ -180,7 +180,7 @@ impl Compiler {
for (i, statement) in program.statements.iter().enumerate() {
let is_last = i == program.statements.len() - 1;
if let ast::Statement::Expression { ref expression } = statement.node {
if let ast::StatementType::Expression { ref expression } = statement.node {
self.compile_expression(expression)?;
if is_last {
@@ -209,12 +209,12 @@ impl Compiler {
// Compile statement in eval mode:
fn compile_statement_eval(
&mut self,
statements: &[ast::LocatedStatement],
statements: &[ast::Statement],
symbol_table: SymbolScope,
) -> Result<(), CompileError> {
self.scope_stack.push(symbol_table);
for statement in statements {
if let ast::Statement::Expression { ref expression } = statement.node {
if let ast::StatementType::Expression { ref expression } = statement.node {
self.compile_expression(expression)?;
} else {
return Err(CompileError {
@@ -227,10 +227,7 @@ impl Compiler {
Ok(())
}
fn compile_statements(
&mut self,
statements: &[ast::LocatedStatement],
) -> Result<(), CompileError> {
fn compile_statements(&mut self, statements: &[ast::Statement]) -> Result<(), CompileError> {
for statement in statements {
self.compile_statement(statement)?
}
@@ -238,11 +235,13 @@ impl Compiler {
}
fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
let role = self.lookup_name(name);
match role {
SymbolRole::Global => bytecode::NameScope::Global,
SymbolRole::Nonlocal => bytecode::NameScope::NonLocal,
_ => bytecode::NameScope::Local,
let symbol = self.lookup_name(name);
if symbol.is_global {
bytecode::NameScope::Global
} else if symbol.is_nonlocal {
bytecode::NameScope::NonLocal
} else {
bytecode::NameScope::Local
}
}
@@ -262,12 +261,13 @@ impl Compiler {
});
}
fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> {
fn compile_statement(&mut self, statement: &ast::Statement) -> Result<(), CompileError> {
trace!("Compiling {:?}", statement);
self.set_source_location(&statement.location);
use ast::StatementType::*;
match &statement.node {
ast::Statement::Import { names } => {
Import { names } => {
// import a, b, c as d
for name in names {
self.emit(Instruction::Import {
@@ -283,7 +283,7 @@ impl Compiler {
}
}
}
ast::Statement::ImportFrom {
ImportFrom {
level,
module,
names,
@@ -326,16 +326,16 @@ impl Compiler {
self.emit(Instruction::Pop);
}
}
ast::Statement::Expression { expression } => {
Expression { expression } => {
self.compile_expression(expression)?;
// Pop result of stack, since we not use it:
self.emit(Instruction::Pop);
}
ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => {
Global { .. } | Nonlocal { .. } => {
// Handled during symbol table construction.
}
ast::Statement::If { test, body, orelse } => {
If { test, body, orelse } => {
let end_label = self.new_label();
match orelse {
None => {
@@ -358,7 +358,7 @@ impl Compiler {
}
self.set_label(end_label);
}
ast::Statement::While { test, body, orelse } => {
While { test, body, orelse } => {
let start_label = self.new_label();
let else_label = self.new_label();
let end_label = self.new_label();
@@ -385,7 +385,7 @@ impl Compiler {
}
self.set_label(end_label);
}
ast::Statement::With { items, body } => {
With { items, body } => {
let end_label = self.new_label();
for item in items {
self.compile_expression(&item.context_expr)?;
@@ -406,16 +406,16 @@ impl Compiler {
}
self.set_label(end_label);
}
ast::Statement::For {
For {
target,
iter,
body,
orelse,
} => self.compile_for(target, iter, body, orelse)?,
ast::Statement::AsyncFor { .. } => {
AsyncFor { .. } => {
unimplemented!("async for");
}
ast::Statement::Raise { exception, cause } => match exception {
Raise { exception, cause } => match exception {
Some(value) => {
self.compile_expression(value)?;
match cause {
@@ -432,30 +432,30 @@ impl Compiler {
self.emit(Instruction::Raise { argc: 0 });
}
},
ast::Statement::Try {
Try {
body,
handlers,
orelse,
finalbody,
} => self.compile_try_statement(body, handlers, orelse, finalbody)?,
ast::Statement::FunctionDef {
FunctionDef {
name,
args,
body,
decorator_list,
returns,
} => self.compile_function_def(name, args, body, decorator_list, returns)?,
ast::Statement::AsyncFunctionDef { .. } => {
AsyncFunctionDef { .. } => {
unimplemented!("async def");
}
ast::Statement::ClassDef {
ClassDef {
name,
body,
bases,
keywords,
decorator_list,
} => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
ast::Statement::Assert { test, msg } => {
Assert { test, msg } => {
// if some flag, ignore all assert statements!
if self.optimize == 0 {
let end_label = self.new_label();
@@ -481,7 +481,7 @@ impl Compiler {
self.set_label(end_label);
}
}
ast::Statement::Break => {
Break => {
if !self.in_loop {
return Err(CompileError {
error: CompileErrorType::InvalidBreak,
@@ -490,7 +490,7 @@ impl Compiler {
}
self.emit(Instruction::Break);
}
ast::Statement::Continue => {
Continue => {
if !self.in_loop {
return Err(CompileError {
error: CompileErrorType::InvalidContinue,
@@ -499,7 +499,7 @@ impl Compiler {
}
self.emit(Instruction::Continue);
}
ast::Statement::Return { value } => {
Return { value } => {
if !self.in_function_def {
return Err(CompileError {
error: CompileErrorType::InvalidReturn,
@@ -519,7 +519,7 @@ impl Compiler {
self.emit(Instruction::ReturnValue);
}
ast::Statement::Assign { targets, value } => {
Assign { targets, value } => {
self.compile_expression(value)?;
for (i, target) in targets.iter().enumerate() {
@@ -529,7 +529,7 @@ impl Compiler {
self.compile_store(target)?;
}
}
ast::Statement::AugAssign { target, op, value } => {
AugAssign { target, op, value } => {
self.compile_expression(target)?;
self.compile_expression(value)?;
@@ -537,12 +537,12 @@ impl Compiler {
self.compile_op(op, true);
self.compile_store(target)?;
}
ast::Statement::Delete { targets } => {
Delete { targets } => {
for target in targets {
self.compile_delete(target)?;
}
}
ast::Statement::Pass => {
Pass => {
self.emit(Instruction::Pass);
}
}
@@ -550,24 +550,24 @@ impl Compiler {
}
fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
match expression {
ast::Expression::Identifier { name } => {
match &expression.node {
ast::ExpressionType::Identifier { name } => {
self.emit(Instruction::DeleteName {
name: name.to_string(),
});
}
ast::Expression::Attribute { value, name } => {
ast::ExpressionType::Attribute { value, name } => {
self.compile_expression(value)?;
self.emit(Instruction::DeleteAttr {
name: name.to_string(),
});
}
ast::Expression::Subscript { a, b } => {
ast::ExpressionType::Subscript { a, b } => {
self.compile_expression(a)?;
self.compile_expression(b)?;
self.emit(Instruction::DeleteSubscript);
}
ast::Expression::Tuple { elements } => {
ast::ExpressionType::Tuple { elements } => {
for element in elements {
self.compile_delete(element)?;
}
@@ -663,10 +663,10 @@ impl Compiler {
fn compile_try_statement(
&mut self,
body: &[ast::LocatedStatement],
body: &[ast::Statement],
handlers: &[ast::ExceptHandler],
orelse: &Option<Vec<ast::LocatedStatement>>,
finalbody: &Option<Vec<ast::LocatedStatement>>,
orelse: &Option<Vec<ast::Statement>>,
finalbody: &Option<Vec<ast::Statement>>,
) -> Result<(), CompileError> {
let mut handler_label = self.new_label();
let finally_label = self.new_label();
@@ -764,7 +764,7 @@ impl Compiler {
&mut self,
name: &str,
args: &ast::Parameters,
body: &[ast::LocatedStatement],
body: &[ast::Statement],
decorator_list: &[ast::Expression],
returns: &Option<ast::Expression>, // TODO: use type hint somehow..
) -> Result<(), CompileError> {
@@ -858,7 +858,7 @@ impl Compiler {
fn compile_class_def(
&mut self,
name: &str,
body: &[ast::LocatedStatement],
body: &[ast::Statement],
bases: &[ast::Expression],
keywords: &[ast::Keyword],
decorator_list: &[ast::Expression],
@@ -989,8 +989,8 @@ impl Compiler {
&mut self,
target: &ast::Expression,
iter: &ast::Expression,
body: &[ast::LocatedStatement],
orelse: &Option<Vec<ast::LocatedStatement>>,
body: &[ast::Statement],
orelse: &Option<Vec<ast::Statement>>,
) -> Result<(), CompileError> {
// Start loop
let start_label = self.new_label();
@@ -1104,27 +1104,27 @@ impl Compiler {
}
fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
match target {
ast::Expression::Identifier { name } => {
match &target.node {
ast::ExpressionType::Identifier { name } => {
self.store_name(name);
}
ast::Expression::Subscript { a, b } => {
ast::ExpressionType::Subscript { a, b } => {
self.compile_expression(a)?;
self.compile_expression(b)?;
self.emit(Instruction::StoreSubscript);
}
ast::Expression::Attribute { value, name } => {
ast::ExpressionType::Attribute { value, name } => {
self.compile_expression(value)?;
self.emit(Instruction::StoreAttr {
name: name.to_string(),
});
}
ast::Expression::List { elements } | ast::Expression::Tuple { elements } => {
ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => {
let mut seen_star = false;
// Scan for star args:
for (i, element) in elements.iter().enumerate() {
if let ast::Expression::Starred { .. } = element {
if let ast::ExpressionType::Starred { .. } = &element.node {
if seen_star {
return Err(CompileError {
error: CompileErrorType::StarArgs,
@@ -1147,7 +1147,7 @@ impl Compiler {
}
for element in elements {
if let ast::Expression::Starred { value } = element {
if let ast::ExpressionType::Starred { value } = &element.node {
self.compile_store(value)?;
} else {
self.compile_store(element)?;
@@ -1192,8 +1192,8 @@ impl Compiler {
context: EvalContext,
) -> Result<(), CompileError> {
// Compile expression for test, and jump to label if false
match expression {
ast::Expression::BoolOp { a, op, b } => match op {
match &expression.node {
ast::ExpressionType::BoolOp { a, op, b } => match op {
ast::BooleanOperator::And => {
let f = false_label.unwrap_or_else(|| self.new_label());
self.compile_test(a, None, Some(f), context)?;
@@ -1246,23 +1246,27 @@ impl Compiler {
fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
trace!("Compiling {:?}", expression);
match expression {
ast::Expression::Call {
use ast::ExpressionType::*;
match &expression.node {
Call {
function,
args,
keywords,
} => self.compile_call(function, args, keywords)?,
ast::Expression::BoolOp { .. } => {
self.compile_test(expression, None, None, EvalContext::Expression)?
}
ast::Expression::Binop { a, op, b } => {
BoolOp { .. } => self.compile_test(
expression,
Option::None,
Option::None,
EvalContext::Expression,
)?,
Binop { a, op, b } => {
self.compile_expression(a)?;
self.compile_expression(b)?;
// Perform operation:
self.compile_op(op, false);
}
ast::Expression::Subscript { a, b } => {
Subscript { a, b } => {
self.compile_expression(a)?;
self.compile_expression(b)?;
self.emit(Instruction::BinaryOperation {
@@ -1270,7 +1274,7 @@ impl Compiler {
inplace: false,
});
}
ast::Expression::Unop { op, a } => {
Unop { op, a } => {
self.compile_expression(a)?;
// Perform operation:
@@ -1283,16 +1287,16 @@ impl Compiler {
let i = Instruction::UnaryOperation { op: i };
self.emit(i);
}
ast::Expression::Attribute { value, name } => {
Attribute { value, name } => {
self.compile_expression(value)?;
self.emit(Instruction::LoadAttr {
name: name.to_string(),
});
}
ast::Expression::Compare { vals, ops } => {
Compare { vals, ops } => {
self.compile_chained_comparison(vals, ops)?;
}
ast::Expression::Number { value } => {
Number { value } => {
let const_value = match value {
ast::Number::Integer { value } => bytecode::Constant::Integer {
value: value.clone(),
@@ -1304,7 +1308,7 @@ impl Compiler {
};
self.emit(Instruction::LoadConst { value: const_value });
}
ast::Expression::List { elements } => {
List { elements } => {
let size = elements.len();
let must_unpack = self.gather_elements(elements)?;
self.emit(Instruction::BuildList {
@@ -1312,7 +1316,7 @@ impl Compiler {
unpack: must_unpack,
});
}
ast::Expression::Tuple { elements } => {
Tuple { elements } => {
let size = elements.len();
let must_unpack = self.gather_elements(elements)?;
self.emit(Instruction::BuildTuple {
@@ -1320,7 +1324,7 @@ impl Compiler {
unpack: must_unpack,
});
}
ast::Expression::Set { elements } => {
Set { elements } => {
let size = elements.len();
let must_unpack = self.gather_elements(elements)?;
self.emit(Instruction::BuildSet {
@@ -1328,7 +1332,7 @@ impl Compiler {
unpack: must_unpack,
});
}
ast::Expression::Dict { elements } => {
Dict { elements } => {
let size = elements.len();
let has_double_star = elements.iter().any(|e| e.0.is_none());
for (key, value) in elements {
@@ -1351,14 +1355,14 @@ impl Compiler {
unpack: has_double_star,
});
}
ast::Expression::Slice { elements } => {
Slice { elements } => {
let size = elements.len();
for element in elements {
self.compile_expression(element)?;
}
self.emit(Instruction::BuildSlice { size });
}
ast::Expression::Yield { value } => {
Yield { value } => {
if !self.in_function_def {
return Err(CompileError {
error: CompileErrorType::InvalidYield,
@@ -1368,16 +1372,16 @@ impl Compiler {
self.mark_generator();
match value {
Some(expression) => self.compile_expression(expression)?,
None => self.emit(Instruction::LoadConst {
Option::None => self.emit(Instruction::LoadConst {
value: bytecode::Constant::None,
}),
};
self.emit(Instruction::YieldValue);
}
ast::Expression::Await { .. } => {
Await { .. } => {
unimplemented!("await");
}
ast::Expression::YieldFrom { value } => {
YieldFrom { value } => {
self.mark_generator();
self.compile_expression(value)?;
self.emit(Instruction::GetIter);
@@ -1386,40 +1390,40 @@ impl Compiler {
});
self.emit(Instruction::YieldFrom);
}
ast::Expression::True => {
True => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::Boolean { value: true },
});
}
ast::Expression::False => {
False => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::Boolean { value: false },
});
}
ast::Expression::None => {
None => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::None,
});
}
ast::Expression::Ellipsis => {
Ellipsis => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::Ellipsis,
});
}
ast::Expression::String { value } => {
String { value } => {
self.compile_string(value)?;
}
ast::Expression::Bytes { value } => {
Bytes { value } => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::Bytes {
value: value.clone(),
},
});
}
ast::Expression::Identifier { name } => {
Identifier { name } => {
self.load_name(name);
}
ast::Expression::Lambda { args, body } => {
Lambda { args, body } => {
let name = "<lambda>".to_string();
// no need to worry about the self.loop_depth because there are no loops in lambda expressions
let flags = self.enter_function(&name, args)?;
@@ -1438,18 +1442,18 @@ impl Compiler {
// Turn code object into function object:
self.emit(Instruction::MakeFunction { flags });
}
ast::Expression::Comprehension { kind, generators } => {
Comprehension { kind, generators } => {
self.compile_comprehension(kind, generators)?;
}
ast::Expression::Starred { value } => {
Starred { value } => {
self.compile_expression(value)?;
self.emit(Instruction::Unpack);
panic!("We should not just unpack a starred args, since the size is unknown.");
}
ast::Expression::IfExpression { test, body, orelse } => {
IfExpression { test, body, orelse } => {
let no_label = self.new_label();
let end_label = self.new_label();
self.compile_test(test, None, None, EvalContext::Expression)?;
self.compile_test(test, Option::None, Option::None, EvalContext::Expression)?;
self.emit(Instruction::JumpIfFalse { target: no_label });
// True case
self.compile_expression(body)?;
@@ -1557,7 +1561,7 @@ impl Compiler {
fn gather_elements(&mut self, elements: &[ast::Expression]) -> Result<bool, CompileError> {
// First determine if we have starred elements:
let has_stars = elements.iter().any(|e| {
if let ast::Expression::Starred { .. } = e {
if let ast::ExpressionType::Starred { .. } = &e.node {
true
} else {
false
@@ -1565,7 +1569,7 @@ impl Compiler {
});
for element in elements {
if let ast::Expression::Starred { value } = element {
if let ast::ExpressionType::Starred { value } = &element.node {
self.compile_expression(value)?;
} else {
self.compile_expression(element)?;
@@ -1792,7 +1796,7 @@ impl Compiler {
assert!(scope.sub_scopes.is_empty());
}
fn lookup_name(&self, name: &str) -> &SymbolRole {
fn lookup_name(&self, name: &str) -> &Symbol {
// println!("Looking up {:?}", name);
let scope = self.scope_stack.last().unwrap();
scope.lookup(name).unwrap()
@@ -1846,10 +1850,10 @@ impl Compiler {
}
}
fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option<String>) {
fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
if let Some(val) = body.get(0) {
if let ast::Statement::Expression { ref expression } = val.node {
if let ast::Expression::String { ref value } = expression {
if let ast::StatementType::Expression { ref expression } = val.node {
if let ast::ExpressionType::String { value } = &expression.node {
if let ast::StringGroup::Constant { ref value } = value {
if let Some((_, body_rest)) = body.split_first() {
return (body_rest, Some(value.to_string()));
compiler/src/symboltable.rs
+152 -118
View File
@@ -24,7 +24,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTa
}
pub fn statements_to_symbol_table(
statements: &[ast::LocatedStatement],
statements: &[ast::Statement],
) -> Result<SymbolScope, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default();
builder.enter_scope();
@@ -36,25 +36,46 @@ pub fn statements_to_symbol_table(
Ok(symbol_table)
}
#[derive(Debug, Clone)]
pub enum SymbolRole {
Global,
Nonlocal,
Used,
Assigned,
}
/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
#[derive(Clone)]
#[derive(Clone, Default)]
pub struct SymbolScope {
/// A set of symbols present on this scope level.
pub symbols: IndexMap<String, SymbolRole>,
pub symbols: IndexMap<String, Symbol>,
/// A list of subscopes in the order as found in the
/// AST nodes.
pub sub_scopes: Vec<SymbolScope>,
}
#[derive(Debug, Clone)]
pub struct Symbol {
pub name: String,
pub is_global: bool,
pub is_local: bool,
pub is_nonlocal: bool,
pub is_param: bool,
pub is_referenced: bool,
pub is_assigned: bool,
pub is_parameter: bool,
pub is_free: bool,
}
impl Symbol {
fn new(name: &str) -> Self {
Symbol {
name: name.to_string(),
is_global: false,
is_local: false,
is_nonlocal: false,
is_param: false,
is_referenced: false,
is_assigned: false,
is_parameter: false,
is_free: false,
}
}
}
#[derive(Debug)]
pub struct SymbolTableError {
error: String,
@@ -73,20 +94,11 @@ impl From<SymbolTableError> for CompileError {
type SymbolTableResult = Result<(), SymbolTableError>;
impl SymbolScope {
pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
pub fn lookup(&self, name: &str) -> Option<&Symbol> {
self.symbols.get(name)
}
}
impl Default for SymbolScope {
fn default() -> Self {
SymbolScope {
symbols: Default::default(),
sub_scopes: Default::default(),
}
}
}
impl std::fmt::Debug for SymbolScope {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
@@ -111,74 +123,73 @@ fn analyze_symbol_table(
}
// Analyze symbols:
for (symbol_name, symbol_role) in &symbol_scope.symbols {
analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?;
for symbol in symbol_scope.symbols.values() {
analyze_symbol(symbol, parent_symbol_scope)?;
}
Ok(())
}
#[allow(clippy::single_match)]
fn analyze_symbol(
symbol_name: &str,
symbol_role: &SymbolRole,
parent_symbol_scope: Option<&SymbolScope>,
) -> SymbolTableResult {
match symbol_role {
SymbolRole::Nonlocal => {
// check if name is defined in parent scope!
if let Some(parent_symbol_scope) = parent_symbol_scope {
if !parent_symbol_scope.symbols.contains_key(symbol_name) {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol_name),
location: Default::default(),
});
}
} else {
fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult {
if symbol.is_nonlocal {
// check if name is defined in parent scope!
if let Some(parent_symbol_scope) = parent_symbol_scope {
if !parent_symbol_scope.symbols.contains_key(&symbol.name) {
return Err(SymbolTableError {
error: format!(
"nonlocal {} defined at place without an enclosing scope",
symbol_name
),
error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(),
});
}
} else {
return Err(SymbolTableError {
error: format!(
"nonlocal {} defined at place without an enclosing scope",
symbol.name
),
location: Default::default(),
});
}
// TODO: add more checks for globals
_ => {}
}
// TODO: add more checks for globals
Ok(())
}
pub struct SymbolTableBuilder {
// Scope stack.
pub scopes: Vec<SymbolScope>,
#[derive(Debug, Clone)]
enum SymbolRole {
Global,
Nonlocal,
Used,
Assigned,
}
impl Default for SymbolTableBuilder {
fn default() -> Self {
SymbolTableBuilder { scopes: vec![] }
}
#[derive(Default)]
struct SymbolTableBuilder {
// Scope stack.
scopes: Vec<SymbolScope>,
}
impl SymbolTableBuilder {
pub fn enter_scope(&mut self) {
fn enter_scope(&mut self) {
let scope = Default::default();
self.scopes.push(scope);
// self.work_scopes.push(Default::default());
}
fn leave_scope(&mut self) {
// Pop scope and add to subscopes of parent scope.
// let work_scope = self.work_scopes.pop().unwrap();
let scope = self.scopes.pop().unwrap();
self.scopes.last_mut().unwrap().sub_scopes.push(scope);
}
pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
self.scan_statements(&program.statements)?;
Ok(())
}
pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult {
fn scan_statements(&mut self, statements: &[ast::Statement]) -> SymbolTableResult {
for statement in statements {
self.scan_statement(statement)?;
}
@@ -210,26 +221,27 @@ impl SymbolTableBuilder {
Ok(())
}
fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult {
fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
use ast::StatementType::*;
match &statement.node {
ast::Statement::Global { names } => {
Global { names } => {
for name in names {
self.register_name(name, SymbolRole::Global)?;
}
}
ast::Statement::Nonlocal { names } => {
Nonlocal { names } => {
for name in names {
self.register_name(name, SymbolRole::Nonlocal)?;
}
}
ast::Statement::FunctionDef {
FunctionDef {
name,
body,
args,
decorator_list,
returns,
}
| ast::Statement::AsyncFunctionDef {
| AsyncFunctionDef {
name,
body,
args,
@@ -247,7 +259,7 @@ impl SymbolTableBuilder {
}
self.leave_scope();
}
ast::Statement::ClassDef {
ClassDef {
name,
body,
bases,
@@ -264,21 +276,21 @@ impl SymbolTableBuilder {
}
self.scan_expressions(decorator_list)?;
}
ast::Statement::Expression { expression } => self.scan_expression(expression)?,
ast::Statement::If { test, body, orelse } => {
Expression { expression } => self.scan_expression(expression)?,
If { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
}
}
ast::Statement::For {
For {
target,
iter,
body,
orelse,
}
| ast::Statement::AsyncFor {
| AsyncFor {
target,
iter,
body,
@@ -291,17 +303,17 @@ impl SymbolTableBuilder {
self.scan_statements(code)?;
}
}
ast::Statement::While { test, body, orelse } => {
While { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
}
}
ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
Break | Continue | Pass => {
// No symbols here.
}
ast::Statement::Import { names } | ast::Statement::ImportFrom { names, .. } => {
Import { names } | ImportFrom { names, .. } => {
for name in names {
if let Some(alias) = &name.alias {
// `import mymodule as myalias`
@@ -312,29 +324,29 @@ impl SymbolTableBuilder {
}
}
}
ast::Statement::Return { value } => {
Return { value } => {
if let Some(expression) = value {
self.scan_expression(expression)?;
}
}
ast::Statement::Assert { test, msg } => {
Assert { test, msg } => {
self.scan_expression(test)?;
if let Some(expression) = msg {
self.scan_expression(expression)?;
}
}
ast::Statement::Delete { targets } => {
Delete { targets } => {
self.scan_expressions(targets)?;
}
ast::Statement::Assign { targets, value } => {
Assign { targets, value } => {
self.scan_expressions(targets)?;
self.scan_expression(value)?;
}
ast::Statement::AugAssign { target, value, .. } => {
AugAssign { target, value, .. } => {
self.scan_expression(target)?;
self.scan_expression(value)?;
}
ast::Statement::With { items, body } => {
With { items, body } => {
for item in items {
self.scan_expression(&item.context_expr)?;
if let Some(expression) = &item.optional_vars {
@@ -343,7 +355,7 @@ impl SymbolTableBuilder {
}
self.scan_statements(body)?;
}
ast::Statement::Try {
Try {
body,
handlers,
orelse,
@@ -366,7 +378,7 @@ impl SymbolTableBuilder {
self.scan_statements(code)?;
}
}
ast::Statement::Raise { exception, cause } => {
Raise { exception, cause } => {
if let Some(expression) = exception {
self.scan_expression(expression)?;
}
@@ -386,26 +398,27 @@ impl SymbolTableBuilder {
}
fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
match expression {
ast::Expression::Binop { a, b, .. } => {
use ast::ExpressionType::*;
match &expression.node {
Binop { a, b, .. } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
}
ast::Expression::BoolOp { a, b, .. } => {
BoolOp { a, b, .. } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
}
ast::Expression::Compare { vals, .. } => {
Compare { vals, .. } => {
self.scan_expressions(vals)?;
}
ast::Expression::Subscript { a, b } => {
Subscript { a, b } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
}
ast::Expression::Attribute { value, .. } => {
Attribute { value, .. } => {
self.scan_expression(value)?;
}
ast::Expression::Dict { elements } => {
Dict { elements } => {
for (key, value) in elements {
if let Some(key) = key {
self.scan_expression(key)?;
@@ -415,36 +428,30 @@ impl SymbolTableBuilder {
self.scan_expression(value)?;
}
}
ast::Expression::Await { value } => {
Await { value } => {
self.scan_expression(value)?;
}
ast::Expression::Yield { value } => {
Yield { value } => {
if let Some(expression) = value {
self.scan_expression(expression)?;
}
}
ast::Expression::YieldFrom { value } => {
YieldFrom { value } => {
self.scan_expression(value)?;
}
ast::Expression::Unop { a, .. } => {
Unop { a, .. } => {
self.scan_expression(a)?;
}
ast::Expression::True
| ast::Expression::False
| ast::Expression::None
| ast::Expression::Ellipsis => {}
ast::Expression::Number { .. } => {}
ast::Expression::Starred { value } => {
True | False | None | Ellipsis => {}
Number { .. } => {}
Starred { value } => {
self.scan_expression(value)?;
}
ast::Expression::Bytes { .. } => {}
ast::Expression::Tuple { elements }
| ast::Expression::Set { elements }
| ast::Expression::List { elements }
| ast::Expression::Slice { elements } => {
Bytes { .. } => {}
Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
self.scan_expressions(elements)?;
}
ast::Expression::Comprehension { kind, generators } => {
Comprehension { kind, generators } => {
match **kind {
ast::ComprehensionKind::GeneratorExpression { ref element }
| ast::ComprehensionKind::List { ref element }
@@ -465,7 +472,7 @@ impl SymbolTableBuilder {
}
}
}
ast::Expression::Call {
Call {
function,
args,
keywords,
@@ -476,18 +483,18 @@ impl SymbolTableBuilder {
self.scan_expression(&keyword.value)?;
}
}
ast::Expression::String { value } => {
String { value } => {
self.scan_string_group(value)?;
}
ast::Expression::Identifier { name } => {
Identifier { name } => {
self.register_name(name, SymbolRole::Used)?;
}
ast::Expression::Lambda { args, body } => {
Lambda { args, body } => {
self.enter_function(args)?;
self.scan_expression(body)?;
self.leave_scope();
}
ast::Expression::IfExpression { test, body, orelse } => {
IfExpression { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_expression(body)?;
self.scan_expression(orelse)?;
@@ -549,6 +556,8 @@ impl SymbolTableBuilder {
let scope_depth = self.scopes.len();
let current_scope = self.scopes.last_mut().unwrap();
let location = Default::default();
// Some checks:
if current_scope.symbols.contains_key(name) {
// Role already set..
match role {
@@ -568,22 +577,47 @@ impl SymbolTableBuilder {
// Ok?
}
}
} else {
match role {
SymbolRole::Nonlocal => {
if scope_depth < 2 {
return Err(SymbolTableError {
error: format!("cannot define nonlocal '{}' at top level.", name),
location,
});
}
}
_ => {
// Ok!
}
// Some more checks:
match role {
SymbolRole::Nonlocal => {
if scope_depth < 2 {
return Err(SymbolTableError {
error: format!("cannot define nonlocal '{}' at top level.", name),
location,
});
}
}
current_scope.symbols.insert(name.to_string(), role);
_ => {
// Ok!
}
}
// Insert symbol when required:
if !current_scope.symbols.contains_key(name) {
let symbol = Symbol::new(name);
current_scope.symbols.insert(name.to_string(), symbol);
}
// Set proper flags on symbol:
let symbol = current_scope.symbols.get_mut(name).unwrap();
match role {
SymbolRole::Nonlocal => {
symbol.is_nonlocal = true;
}
SymbolRole::Assigned => {
symbol.is_assigned = true;
// symbol.is_local = true;
}
SymbolRole::Global => {
symbol.is_global = true;
}
SymbolRole::Used => {
symbol.is_referenced = true;
}
}
Ok(())
}
}
crawl_sourcecode.py
+16 -2
View File
@@ -26,7 +26,8 @@ print(t)
shift = 3
def print_node(node, indent=0):
if isinstance(node, ast.AST):
print(' '*indent, "NODE", node.__class__.__name__)
lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else ''
print(' '*indent, "NODE", node.__class__.__name__, lineno)
for field in node._fields:
print(' '*indent,'-', field)
f = getattr(node, field)
@@ -41,12 +42,25 @@ def print_node(node, indent=0):
print_node(t)
# print(ast.dump(t))
flag_names = [
'is_referenced',
'is_assigned',
'is_global',
'is_local',
'is_parameter',
'is_free',
]
def print_table(table, indent=0):
print(' '*indent, 'table:', table.get_name())
print(' '*indent, ' ', 'Syms:')
for sym in table.get_symbols():
print(' '*indent, ' ', sym.get_name(), 'is_referenced=', sym.is_referenced(), 'is_assigned=', sym.is_assigned())
flags = []
for flag_name in flag_names:
func = getattr(sym, flag_name)
if func():
flags.append(flag_name)
print(' '*indent, ' sym:', sym.get_name(), 'flags:', ' '.join(flags))
print(' '*indent, ' ', 'Child tables:')
for child in table.get_children():
print_table(child, indent=indent+shift)
parser/src/ast.rs
+25 -23
View File
@@ -17,13 +17,13 @@ pub struct Node {
#[derive(Debug, PartialEq)]
pub enum Top {
Program(Program),
Statement(Vec<LocatedStatement>),
Statement(Vec<Statement>),
Expression(Expression),
}
#[derive(Debug, PartialEq)]
pub struct Program {
pub statements: Vec<LocatedStatement>,
pub statements: Vec<Statement>,
}
#[derive(Debug, PartialEq)]
@@ -38,12 +38,12 @@ pub struct Located<T> {
pub node: T,
}
pub type LocatedStatement = Located<Statement>;
pub type Statement = Located<StatementType>;
/// Abstract syntax tree nodes for python statements.
#[allow(clippy::large_enum_variant)]
#[derive(Debug, PartialEq)]
pub enum Statement {
pub enum StatementType {
Break,
Continue,
Return {
@@ -85,43 +85,43 @@ pub enum Statement {
},
If {
test: Expression,
body: Vec<LocatedStatement>,
orelse: Option<Vec<LocatedStatement>>,
body: Vec<Statement>,
orelse: Option<Vec<Statement>>,
},
While {
test: Expression,
body: Vec<LocatedStatement>,
orelse: Option<Vec<LocatedStatement>>,
body: Vec<Statement>,
orelse: Option<Vec<Statement>>,
},
With {
items: Vec<WithItem>,
body: Vec<LocatedStatement>,
body: Vec<Statement>,
},
For {
target: Expression,
iter: Expression,
body: Vec<LocatedStatement>,
orelse: Option<Vec<LocatedStatement>>,
body: Vec<Statement>,
orelse: Option<Vec<Statement>>,
},
AsyncFor {
target: Expression,
iter: Expression,
body: Vec<LocatedStatement>,
orelse: Option<Vec<LocatedStatement>>,
body: Vec<Statement>,
orelse: Option<Vec<Statement>>,
},
Raise {
exception: Option<Expression>,
cause: Option<Expression>,
},
Try {
body: Vec<LocatedStatement>,
body: Vec<Statement>,
handlers: Vec<ExceptHandler>,
orelse: Option<Vec<LocatedStatement>>,
finalbody: Option<Vec<LocatedStatement>>,
orelse: Option<Vec<Statement>>,
finalbody: Option<Vec<Statement>>,
},
ClassDef {
name: String,
body: Vec<LocatedStatement>,
body: Vec<Statement>,
bases: Vec<Expression>,
keywords: Vec<Keyword>,
decorator_list: Vec<Expression>,
@@ -129,14 +129,14 @@ pub enum Statement {
FunctionDef {
name: String,
args: Parameters,
body: Vec<LocatedStatement>,
body: Vec<Statement>,
decorator_list: Vec<Expression>,
returns: Option<Expression>,
},
AsyncFunctionDef {
name: String,
args: Parameters,
body: Vec<LocatedStatement>,
body: Vec<Statement>,
decorator_list: Vec<Expression>,
returns: Option<Expression>,
},
@@ -148,8 +148,10 @@ pub struct WithItem {
pub optional_vars: Option<Expression>,
}
pub type Expression = Located<ExpressionType>;
#[derive(Debug, PartialEq)]
pub enum Expression {
pub enum ExpressionType {
BoolOp {
a: Box<Expression>,
op: BooleanOperator,
@@ -242,10 +244,10 @@ pub enum Expression {
impl Expression {
/// Returns a short name for the node suitable for use in error messages.
pub fn name(&self) -> &'static str {
use self::Expression::*;
use self::ExpressionType::*;
use self::StringGroup::*;
match self {
match &self.node {
BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
Subscript { .. } => "subscript",
Await { .. } => "await expression",
@@ -330,7 +332,7 @@ pub struct Keyword {
pub struct ExceptHandler {
pub typ: Option<Expression>,
pub name: Option<String>,
pub body: Vec<LocatedStatement>,
pub body: Vec<Statement>,
}
#[derive(Debug, PartialEq)]
parser/src/fstring.rs
+8 -5
View File
@@ -173,9 +173,12 @@ mod tests {
use super::*;
fn mk_ident(name: &str) -> ast::Expression {
ast::Expression::Identifier {
name: name.to_owned(),
fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression {
ast::Expression {
location: ast::Location::new(row, col),
node: ast::ExpressionType::Identifier {
name: name.to_owned(),
},
}
}
@@ -189,12 +192,12 @@ mod tests {
Joined {
values: vec![
FormattedValue {
value: Box::new(mk_ident("a")),
value: Box::new(mk_ident("a", 1, 1)),
conversion: None,
spec: String::new(),
},
FormattedValue {
value: Box::new(mk_ident("b")),
value: Box::new(mk_ident("b", 1, 1)),
conversion: None,
spec: String::new(),
},
parser/src/parser.rs
+195 -253
View File
@@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
do_lalr_parsing!(source, Program, StartProgram)
}
pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, ParseError> {
pub fn parse_statement(source: &str) -> Result<Vec<ast::Statement>, ParseError> {
do_lalr_parsing!(source, Statement, StartStatement)
}
@@ -46,16 +46,25 @@ pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, Parse
/// extern crate num_bigint;
/// use num_bigint::BigInt;
/// use rustpython_parser::{parser, ast};
/// let expr = parser::parse_expression("1+2").unwrap();
/// let expr = parser::parse_expression("1 + 2").unwrap();
///
/// assert_eq!(ast::Expression::Binop {
/// a: Box::new(ast::Expression::Number {
/// value: ast::Number::Integer { value: BigInt::from(1) }
/// }),
/// op: ast::Operator::Add,
/// b: Box::new(ast::Expression::Number {
/// value: ast::Number::Integer { value: BigInt::from(2) }
/// })
/// assert_eq!(ast::Expression {
/// location: ast::Location::new(1, 3),
/// node: ast::ExpressionType::Binop {
/// a: Box::new(ast::Expression {
/// location: ast::Location::new(1, 1),
/// node: ast::ExpressionType::Number {
/// value: ast::Number::Integer { value: BigInt::from(1) }
/// }
/// }),
/// op: ast::Operator::Add,
/// b: Box::new(ast::Expression {
/// location: ast::Location::new(1, 5),
/// node: ast::ExpressionType::Number {
/// value: ast::Number::Integer { value: BigInt::from(2) }
/// }
/// })
/// }
/// },
/// expr);
///
@@ -72,6 +81,44 @@ mod tests {
use super::parse_statement;
use num_bigint::BigInt;
fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression {
ast::Expression {
location: ast::Location::new(row, col),
node: ast::ExpressionType::Identifier {
name: name.to_owned(),
},
}
}
fn make_int(value: i32, row: usize, col: usize) -> ast::Expression {
ast::Expression {
location: ast::Location::new(row, col),
node: ast::ExpressionType::Number {
value: ast::Number::Integer {
value: BigInt::from(value),
},
},
}
}
fn make_string(value: &str, row: usize, col: usize) -> ast::Expression {
ast::Expression {
location: ast::Location::new(row, col),
node: ast::ExpressionType::String {
value: ast::StringGroup::Constant {
value: String::from(value),
},
},
}
}
fn as_statement(expr: ast::Expression) -> ast::Statement {
ast::Statement {
location: expr.location.clone(),
node: ast::StatementType::Expression { expression: expr },
}
}
#[test]
fn test_parse_empty() {
let parse_ast = parse_program(&String::from("\n"));
@@ -85,19 +132,16 @@ mod tests {
assert_eq!(
parse_ast,
ast::Program {
statements: vec![ast::LocatedStatement {
statements: vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::Expression {
expression: ast::Expression::Call {
function: Box::new(ast::Expression::Identifier {
name: String::from("print"),
}),
args: vec![ast::Expression::String {
value: ast::StringGroup::Constant {
value: String::from("Hello world")
}
}],
keywords: vec![],
node: ast::StatementType::Expression {
expression: ast::Expression {
location: ast::Location::new(1, 6),
node: ast::ExpressionType::Call {
function: Box::new(mk_ident("print", 1, 1)),
args: vec![make_string("Hello world", 1, 8)],
keywords: vec![],
}
},
},
},],
@@ -112,26 +156,16 @@ mod tests {
assert_eq!(
parse_ast,
ast::Program {
statements: vec![ast::LocatedStatement {
statements: vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::Expression {
expression: ast::Expression::Call {
function: Box::new(ast::Expression::Identifier {
name: String::from("print"),
}),
args: vec![
ast::Expression::String {
value: ast::StringGroup::Constant {
value: String::from("Hello world"),
}
},
ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(2)
},
}
],
keywords: vec![],
node: ast::StatementType::Expression {
expression: ast::Expression {
location: ast::Location::new(1, 6),
node: ast::ExpressionType::Call {
function: Box::new(mk_ident("print", 1, 1)),
args: vec![make_string("Hello world", 1, 8), make_int(2, 1, 22),],
keywords: vec![],
},
},
},
},],
@@ -146,26 +180,19 @@ mod tests {
assert_eq!(
parse_ast,
ast::Program {
statements: vec![ast::LocatedStatement {
statements: vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::Expression {
expression: ast::Expression::Call {
function: Box::new(ast::Expression::Identifier {
name: String::from("my_func"),
}),
args: vec![ast::Expression::String {
value: ast::StringGroup::Constant {
value: String::from("positional"),
}
}],
keywords: vec![ast::Keyword {
name: Some("keyword".to_string()),
value: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(2)
},
}
}],
node: ast::StatementType::Expression {
expression: ast::Expression {
location: ast::Location::new(1, 8),
node: ast::ExpressionType::Call {
function: Box::new(mk_ident("my_func", 1, 1)),
args: vec![make_string("positional", 1, 10)],
keywords: vec![ast::Keyword {
name: Some("keyword".to_string()),
value: make_int(2, 1, 31),
}],
}
},
},
},],
@@ -179,52 +206,17 @@ mod tests {
let parse_ast = parse_statement(&source).unwrap();
assert_eq!(
parse_ast,
vec![ast::LocatedStatement {
vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::If {
test: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(1)
},
},
body: vec![ast::LocatedStatement {
location: ast::Location::new(1, 7),
node: ast::Statement::Expression {
expression: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(10)
},
}
},
},],
orelse: Some(vec![ast::LocatedStatement {
node: ast::StatementType::If {
test: make_int(1, 1, 4),
body: vec![as_statement(make_int(10, 1, 7))],
orelse: Some(vec![ast::Statement {
location: ast::Location::new(2, 1),
node: ast::Statement::If {
test: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(2)
},
},
body: vec![ast::LocatedStatement {
location: ast::Location::new(2, 9),
node: ast::Statement::Expression {
expression: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(20)
},
},
},
},],
orelse: Some(vec![ast::LocatedStatement {
location: ast::Location::new(3, 7),
node: ast::Statement::Expression {
expression: ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(30)
},
},
},
},]),
node: ast::StatementType::If {
test: make_int(2, 2, 6),
body: vec![as_statement(make_int(20, 2, 9))],
orelse: Some(vec![as_statement(make_int(30, 3, 7))]),
}
},]),
}
@@ -238,39 +230,36 @@ mod tests {
let parse_ast = parse_statement(&source);
assert_eq!(
parse_ast,
Ok(vec![ast::LocatedStatement {
Ok(vec![as_statement(ast::Expression {
location: ast::Location::new(1, 1),
node: ast::Statement::Expression {
expression: ast::Expression::Lambda {
args: ast::Parameters {
args: vec![
ast::Parameter {
arg: String::from("x"),
annotation: None,
},
ast::Parameter {
arg: String::from("y"),
annotation: None,
}
],
kwonlyargs: vec![],
vararg: ast::Varargs::None,
kwarg: ast::Varargs::None,
defaults: vec![],
kw_defaults: vec![],
},
body: Box::new(ast::Expression::Binop {
a: Box::new(ast::Expression::Identifier {
name: String::from("x"),
}),
node: ast::ExpressionType::Lambda {
args: ast::Parameters {
args: vec![
ast::Parameter {
arg: String::from("x"),
annotation: None,
},
ast::Parameter {
arg: String::from("y"),
annotation: None,
}
],
kwonlyargs: vec![],
vararg: ast::Varargs::None,
kwarg: ast::Varargs::None,
defaults: vec![],
kw_defaults: vec![],
},
body: Box::new(ast::Expression {
location: ast::Location::new(1, 16),
node: ast::ExpressionType::Binop {
a: Box::new(mk_ident("x", 1, 14)),
op: ast::Operator::Mult,
b: Box::new(ast::Expression::Identifier {
name: String::from("y"),
})
})
}
b: Box::new(mk_ident("y", 1, 18))
}
})
}
}])
})])
)
}
@@ -280,32 +269,20 @@ mod tests {
assert_eq!(
parse_statement(&source),
Ok(vec![ast::LocatedStatement {
Ok(vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::Assign {
targets: vec![ast::Expression::Tuple {
elements: vec![
ast::Expression::Identifier {
name: "a".to_string()
},
ast::Expression::Identifier {
name: "b".to_string()
}
]
node: ast::StatementType::Assign {
targets: vec![ast::Expression {
location: ast::Location::new(1, 1),
node: ast::ExpressionType::Tuple {
elements: vec![mk_ident("a", 1, 1), mk_ident("b", 1, 4),]
}
}],
value: ast::Expression::Tuple {
elements: vec![
ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(4)
}
},
ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(5)
}
}
]
value: ast::Expression {
location: ast::Location::new(1, 8),
node: ast::ExpressionType::Tuple {
elements: vec![make_int(4, 1, 8), make_int(5, 1, 11),]
}
}
}
}])
@@ -319,23 +296,16 @@ mod tests {
);
assert_eq!(
parse_statement(&source),
Ok(vec![ast::LocatedStatement {
Ok(vec![ast::Statement {
location: ast::Location::new(1, 1),
node: ast::Statement::ClassDef {
node: ast::StatementType::ClassDef {
name: String::from("Foo"),
bases: vec![
ast::Expression::Identifier {
name: String::from("A")
},
ast::Expression::Identifier {
name: String::from("B")
}
],
bases: vec![mk_ident("A", 1, 11), mk_ident("B", 1, 14)],
keywords: vec![],
body: vec![
ast::LocatedStatement {
ast::Statement {
location: ast::Location::new(2, 2),
node: ast::Statement::FunctionDef {
node: ast::StatementType::FunctionDef {
name: String::from("__init__"),
args: ast::Parameters {
args: vec![ast::Parameter {
@@ -348,17 +318,17 @@ mod tests {
defaults: vec![],
kw_defaults: vec![],
},
body: vec![ast::LocatedStatement {
body: vec![ast::Statement {
location: ast::Location::new(3, 3),
node: ast::Statement::Pass,
node: ast::StatementType::Pass,
}],
decorator_list: vec![],
returns: None,
}
},
ast::LocatedStatement {
ast::Statement {
location: ast::Location::new(4, 2),
node: ast::Statement::FunctionDef {
node: ast::StatementType::FunctionDef {
name: String::from("method_with_default"),
args: ast::Parameters {
args: vec![
@@ -374,16 +344,12 @@ mod tests {
kwonlyargs: vec![],
vararg: ast::Varargs::None,
kwarg: ast::Varargs::None,
defaults: vec![ast::Expression::String {
value: ast::StringGroup::Constant {
value: "default".to_string()
}
}],
defaults: vec![make_string("default", 4, 37)],
kw_defaults: vec![],
},
body: vec![ast::LocatedStatement {
body: vec![ast::Statement {
location: ast::Location::new(5, 3),
node: ast::Statement::Pass,
node: ast::StatementType::Pass,
}],
decorator_list: vec![],
returns: None,
@@ -402,21 +368,18 @@ mod tests {
let parse_ast = parse_expression(&source).unwrap();
assert_eq!(
parse_ast,
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::List {
element: ast::Expression::Identifier {
name: "x".to_string()
}
}),
generators: vec![ast::Comprehension {
target: ast::Expression::Identifier {
name: "y".to_string()
},
iter: ast::Expression::Identifier {
name: "z".to_string()
},
ifs: vec![],
}],
ast::Expression {
location: ast::Location::new(1, 2),
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::List {
element: mk_ident("x", 1, 2),
}),
generators: vec![ast::Comprehension {
target: mk_ident("y", 1, 8),
iter: mk_ident("z", 1, 13),
ifs: vec![],
}],
}
}
);
}
@@ -427,66 +390,45 @@ mod tests {
let parse_ast = parse_expression(&source).unwrap();
assert_eq!(
parse_ast,
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::List {
element: ast::Expression::Identifier {
name: "x".to_string()
}
}),
generators: vec![
ast::Comprehension {
target: ast::Expression::Tuple {
elements: vec![
ast::Expression::Identifier {
name: "y".to_string()
ast::Expression {
location: ast::Location::new(1, 2),
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::List {
element: mk_ident("x", 1, 2)
}),
generators: vec![
ast::Comprehension {
target: ast::Expression {
location: ast::Location::new(1, 8),
node: ast::ExpressionType::Tuple {
elements: vec![mk_ident("y", 1, 8), mk_ident("y2", 1, 11),],
}
},
iter: mk_ident("z", 1, 17),
ifs: vec![],
},
ast::Comprehension {
target: mk_ident("a", 1, 23),
iter: mk_ident("b", 1, 28),
ifs: vec![
ast::Expression {
location: ast::Location::new(1, 35),
node: ast::ExpressionType::Compare {
vals: vec![mk_ident("a", 1, 33), make_int(5, 1, 37),],
ops: vec![ast::Comparison::Less],
}
},
ast::Expression::Identifier {
name: "y2".to_string()
ast::Expression {
location: ast::Location::new(1, 44),
node: ast::ExpressionType::Compare {
vals: vec![mk_ident("a", 1, 42), make_int(10, 1, 46),],
ops: vec![ast::Comparison::Greater],
},
},
],
},
iter: ast::Expression::Identifier {
name: "z".to_string()
},
ifs: vec![],
},
ast::Comprehension {
target: ast::Expression::Identifier {
name: "a".to_string()
},
iter: ast::Expression::Identifier {
name: "b".to_string()
},
ifs: vec![
ast::Expression::Compare {
vals: vec![
ast::Expression::Identifier {
name: "a".to_string()
},
ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(5)
}
}
],
ops: vec![ast::Comparison::Less],
},
ast::Expression::Compare {
vals: vec![
ast::Expression::Identifier {
name: "a".to_string()
},
ast::Expression::Number {
value: ast::Number::Integer {
value: BigInt::from(10)
}
}
],
ops: vec![ast::Comparison::Greater],
},
],
}
],
}
],
}
}
);
}
parser/src/python.lalrpop
+356 -219
View File
@@ -31,22 +31,22 @@ Program: ast::Program = {
};
// A file line either has a declaration, or an empty newline:
FileLine: Vec<ast::LocatedStatement> = {
FileLine: Vec<ast::Statement> = {
Statement,
"\n" => vec![],
};
Suite: Vec<ast::LocatedStatement> = {
Suite: Vec<ast::Statement> = {
SimpleStatement,
"\n" indent <s:Statement+> dedent => s.into_iter().flatten().collect(),
};
Statement: Vec<ast::LocatedStatement> = {
Statement: Vec<ast::Statement> = {
SimpleStatement,
<s:CompoundStatement> => vec![s],
};
SimpleStatement: Vec<ast::LocatedStatement> = {
SimpleStatement: Vec<ast::Statement> = {
<s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
let mut statements = vec![s1];
statements.extend(s2.into_iter().map(|e| e.1));
@@ -54,7 +54,7 @@ SimpleStatement: Vec<ast::LocatedStatement> = {
}
};
SmallStatement: ast::LocatedStatement = {
SmallStatement: ast::Statement = {
ExpressionStatement,
PassStatement,
DelStatement,
@@ -65,52 +65,52 @@ SmallStatement: ast::LocatedStatement = {
AssertStatement,
};
PassStatement: ast::LocatedStatement = {
PassStatement: ast::Statement = {
<loc:@L> "pass" => {
ast::LocatedStatement {
ast::Statement {
location: loc,
node: ast::Statement::Pass,
node: ast::StatementType::Pass,
}
},
};
DelStatement: ast::LocatedStatement = {
DelStatement: ast::Statement = {
<loc:@L> "del" <e:ExpressionList2> => {
ast::LocatedStatement {
ast::Statement {
location: loc,
node: ast::Statement::Delete { targets: e },
node: ast::StatementType::Delete { targets: e },
}
},
};
ExpressionStatement: ast::LocatedStatement = {
<loc:@L> <expr:TestOrStarExprList> <suffix:AssignSuffix*> => {
ExpressionStatement: ast::Statement = {
<location:@L> <expr:TestOrStarExprList> <suffix:AssignSuffix*> => {
// Just an expression, no assignment:
if suffix.is_empty() {
ast::LocatedStatement {
location: loc.clone(),
node: ast::Statement::Expression { expression: expr }
ast::Statement {
location,
node: ast::StatementType::Expression { expression: expr }
}
} else {
let mut targets = vec![expr];
let mut values = suffix;
let mut targets = vec![expr];
let mut values = suffix;
while values.len() > 1 {
while values.len() > 1 {
targets.push(values.remove(0));
}
}
let value = values.into_iter().next().unwrap();
let value = values.into_iter().next().unwrap();
ast::LocatedStatement {
location: loc.clone(),
node: ast::Statement::Assign { targets, value },
}
ast::Statement {
location,
node: ast::StatementType::Assign { targets, value },
}
}
},
<loc:@L> <expr:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::AugAssign {
<location:@L> <expr:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
ast::Statement {
location,
node: ast::StatementType::AugAssign {
target: Box::new(expr),
op,
value: Box::new(rhs)
@@ -125,11 +125,14 @@ AssignSuffix: ast::Expression = {
};
TestOrStarExprList: ast::Expression = {
<elements:OneOrMore<TestOrStarExpr>> <comma:","?> => {
<location:@L> <elements:OneOrMore<TestOrStarExpr>> <comma:","?> => {
if elements.len() == 1 && comma.is_none() {
elements.into_iter().next().unwrap()
} else {
ast::Expression::Tuple { elements }
ast::Expression {
location,
node: ast::ExpressionType::Tuple { elements }
}
}
}
};
@@ -155,60 +158,60 @@ AugAssign: ast::Operator = {
"//=" => ast::Operator::FloorDiv,
};
FlowStatement: ast::LocatedStatement = {
<loc:@L> "break" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Break,
FlowStatement: ast::Statement = {
<location:@L> "break" => {
ast::Statement {
location,
node: ast::StatementType::Break,
}
},
<loc:@L> "continue" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Continue,
<location:@L> "continue" => {
ast::Statement {
location,
node: ast::StatementType::Continue,
}
},
<loc:@L> "return" <value:TestList?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Return { value },
<location:@L> "return" <value:TestList?> => {
ast::Statement {
location,
node: ast::StatementType::Return { value },
}
},
<loc:@L> <y:YieldExpr> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Expression { expression: y },
<location:@L> <y:YieldExpr> => {
ast::Statement {
location,
node: ast::StatementType::Expression { expression: y },
}
},
RaiseStatement,
};
RaiseStatement: ast::LocatedStatement = {
<loc:@L> "raise" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Raise { exception: None, cause: None },
RaiseStatement: ast::Statement = {
<location:@L> "raise" => {
ast::Statement {
location,
node: ast::StatementType::Raise { exception: None, cause: None },
}
},
<loc:@L> "raise" <t:Test> <c:("from" Test)?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Raise { exception: Some(t), cause: c.map(|x| x.1) },
<location:@L> "raise" <t:Test> <c:("from" Test)?> => {
ast::Statement {
location,
node: ast::StatementType::Raise { exception: Some(t), cause: c.map(|x| x.1) },
}
},
};
ImportStatement: ast::LocatedStatement = {
<loc:@L> "import" <names: Comma<ImportPart<<DottedName>>>> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Import { names },
ImportStatement: ast::Statement = {
<location:@L> "import" <names: Comma<ImportPart<<DottedName>>>> => {
ast::Statement {
location,
node: ast::StatementType::Import { names },
}
},
<loc:@L> "from" <n:ImportFromLocation> "import" <names: ImportAsNames> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::ImportFrom {
<location:@L> "from" <n:ImportFromLocation> "import" <names: ImportAsNames> => {
ast::Statement {
location,
node: ast::StatementType::ImportFrom {
level: n.0,
module: n.1,
names
@@ -259,36 +262,36 @@ DottedName: String = {
},
};
GlobalStatement: ast::LocatedStatement = {
<loc:@L> "global" <names:OneOrMore<Identifier>> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Global { names }
GlobalStatement: ast::Statement = {
<location:@L> "global" <names:OneOrMore<Identifier>> => {
ast::Statement {
location,
node: ast::StatementType::Global { names }
}
},
};
NonlocalStatement: ast::LocatedStatement = {
<loc:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Nonlocal { names }
NonlocalStatement: ast::Statement = {
<location:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
ast::Statement {
location,
node: ast::StatementType::Nonlocal { names }
}
},
};
AssertStatement: ast::LocatedStatement = {
<loc:@L> "assert" <test:Test> <msg: ("," Test)?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Assert {
AssertStatement: ast::Statement = {
<location:@L> "assert" <test:Test> <msg: ("," Test)?> => {
ast::Statement {
location,
node: ast::StatementType::Assert {
test, msg: msg.map(|e| e.1)
}
}
},
};
CompoundStatement: ast::LocatedStatement = {
CompoundStatement: ast::Statement = {
IfStatement,
WhileStatement,
ForStatement,
@@ -298,58 +301,58 @@ CompoundStatement: ast::LocatedStatement = {
ClassDef,
};
IfStatement: ast::LocatedStatement = {
<loc:@L> "if" <test:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
IfStatement: ast::Statement = {
<location:@L> "if" <test:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
// Determine last else:
let mut last = s3.map(|s| s.2);
// handle elif:
for i in s2.into_iter().rev() {
let x = ast::LocatedStatement {
let x = ast::Statement {
location: i.0,
node: ast::Statement::If { test: i.2, body: i.4, orelse: last },
node: ast::StatementType::If { test: i.2, body: i.4, orelse: last },
};
last = Some(vec![x]);
}
ast::LocatedStatement {
location: loc,
node: ast::Statement::If { test, body: s1, orelse: last }
ast::Statement {
location,
node: ast::StatementType::If { test, body: s1, orelse: last }
}
},
};
WhileStatement: ast::LocatedStatement = {
<loc:@L> "while" <test:Test> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
WhileStatement: ast::Statement = {
<location:@L> "while" <test:Test> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
let or_else = s2.map(|s| s.2);
ast::LocatedStatement {
location: loc,
node: ast::Statement::While { test, body, orelse: or_else },
ast::Statement {
location,
node: ast::StatementType::While { test, body, orelse: or_else },
}
},
};
ForStatement: ast::LocatedStatement = {
<loc:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
ForStatement: ast::Statement = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
let orelse = s2.map(|s| s.2);
ast::LocatedStatement {
location: loc,
ast::Statement {
location,
node: if is_async.is_some() {
ast::Statement::AsyncFor { target, iter, body, orelse }
ast::StatementType::AsyncFor { target, iter, body, orelse }
} else {
ast::Statement::For { target, iter, body, orelse }
ast::StatementType::For { target, iter, body, orelse }
},
}
},
};
TryStatement: ast::LocatedStatement = {
<loc:@L> "try" ":" <body:Suite> <handlers:ExceptClause*> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
TryStatement: ast::Statement = {
<location:@L> "try" ":" <body:Suite> <handlers:ExceptClause*> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
let or_else = else_suite.map(|s| s.2);
let finalbody = finally.map(|s| s.2);
ast::LocatedStatement {
location: loc,
node: ast::Statement::Try {
ast::Statement {
location,
node: ast::StatementType::Try {
body: body,
handlers: handlers,
orelse: or_else,
@@ -376,11 +379,11 @@ ExceptClause: ast::ExceptHandler = {
},
};
WithStatement: ast::LocatedStatement = {
<loc:@L> "with" <items:OneOrMore<WithItem>> ":" <s:Suite> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::With { items: items, body: s },
WithStatement: ast::Statement = {
<location:@L> "with" <items:OneOrMore<WithItem>> ":" <s:Suite> => {
ast::Statement {
location,
node: ast::StatementType::With { items: items, body: s },
}
},
};
@@ -392,12 +395,12 @@ WithItem: ast::WithItem = {
},
};
FuncDef: ast::LocatedStatement = {
<d:Decorator*> <loc:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
ast::LocatedStatement {
location: loc,
FuncDef: ast::Statement = {
<d:Decorator*> <location:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
ast::Statement {
location,
node: if is_async.is_some() {
ast::Statement::AsyncFunctionDef {
ast::StatementType::AsyncFunctionDef {
name: i,
args: a,
body: s,
@@ -405,7 +408,7 @@ FuncDef: ast::LocatedStatement = {
returns: r.map(|x| x.1),
}
} else {
ast::Statement::FunctionDef {
ast::StatementType::FunctionDef {
name: i,
args: a,
body: s,
@@ -552,43 +555,55 @@ KwargParameter<ArgType>: Option<ast::Parameter> = {
}
};
ClassDef: ast::LocatedStatement = {
<d:Decorator*> <loc:@L> "class" <n:Identifier> <a:("(" ArgumentList ")")?> ":" <s:Suite> => {
ClassDef: ast::Statement = {
<d:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
let (bases, keywords) = match a {
Some((_, args, _)) => args,
None => (vec![], vec![]),
};
ast::LocatedStatement {
location: loc,
node: ast::Statement::ClassDef {
name: n,
bases: bases,
keywords: keywords,
body: s,
decorator_list: d,
ast::Statement {
location,
node: ast::StatementType::ClassDef {
name,
bases,
keywords,
body,
decorator_list: d,
},
}
},
};
Path: ast::Expression = {
<n:Identifier> => ast::Expression::Identifier { name: n },
<p:Path> "." <n:name> => {
ast::Expression::Attribute {
value: Box::new(p),
name: n,
<location:@L> <n:Identifier> => ast::Expression {
location,
node: ast::ExpressionType::Identifier { name: n }
},
<p:Path> <location:@L> "." <n:name> => {
ast::Expression {
location,
node: ast::ExpressionType::Attribute {
value: Box::new(p),
name: n,
}
}
},
};
// Decorators:
Decorator: ast::Expression = {
"@" <p:Path> <a: ("(" ArgumentList ")")?> "\n" => {
"@" <p:Path> <a: (@L "(" ArgumentList ")")?> "\n" => {
match a {
Some((_, args, _)) => ast::Expression::Call {
function: Box::new(p),
args: args.0,
keywords: args.1,
Some((location, _, args, _)) => {
let (args, keywords) = args;
ast::Expression {
location,
node: ast::ExpressionType::Call {
function: Box::new(p),
args,
keywords,
}
}
},
None => p,
}
@@ -596,17 +611,26 @@ Decorator: ast::Expression = {
};
YieldExpr: ast::Expression = {
"yield" <value:TestList?> => ast::Expression::Yield { value: value.map(Box::new) },
"yield" "from" <e:Test> => ast::Expression::YieldFrom { value: Box::new(e) },
<location:@L> "yield" <value:TestList?> => ast::Expression {
location,
node: ast::ExpressionType::Yield { value: value.map(Box::new) }
},
<location:@L> "yield" "from" <e:Test> => ast::Expression {
location,
node: ast::ExpressionType::YieldFrom { value: Box::new(e) }
},
};
Test: ast::Expression = {
<expr:OrTest> <condition: ("if" OrTest "else" Test)?> => {
<expr:OrTest> <condition: (@L "if" OrTest "else" Test)?> => {
if let Some(c) = condition {
ast::Expression::IfExpression {
test: Box::new(c.1),
body: Box::new(expr),
orelse: Box::new(c.3),
ast::Expression {
location: c.0,
node: ast::ExpressionType::IfExpression {
test: Box::new(c.2),
body: Box::new(expr),
orelse: Box::new(c.4),
}
}
} else {
expr
@@ -616,37 +640,52 @@ Test: ast::Expression = {
};
LambdaDef: ast::Expression = {
"lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> =>
ast::Expression::Lambda {
args: p.unwrap_or(Default::default()),
body: Box::new(body)
<location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> =>
ast::Expression {
location,
node: ast::ExpressionType::Lambda {
args: p.unwrap_or(Default::default()),
body: Box::new(body)
}
}
}
OrTest: ast::Expression = {
AndTest,
<e1:OrTest> "or" <e2:AndTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) },
<e1:OrTest> <location:@L> "or" <e2:AndTest> => ast::Expression {
location,
node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }
},
};
AndTest: ast::Expression = {
NotTest,
<e1:AndTest> "and" <e2:NotTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) },
<e1:AndTest> <location:@L> "and" <e2:NotTest> => ast::Expression {
location,
node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }
},
};
NotTest: ast::Expression = {
"not" <e:NotTest> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not },
<location:@L> "not" <e:NotTest> => ast::Expression {
location,
node: ast::ExpressionType::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }
},
Comparison,
};
Comparison: ast::Expression = {
<e:Expression> <comparisons:(CompOp Expression)+> => {
<e:Expression> <location:@L> <comparisons:(CompOp Expression)+> => {
let mut vals = vec![e];
let mut ops = vec![];
for x in comparisons {
ops.push(x.0);
vals.push(x.1);
}
ast::Expression::Compare { vals, ops }
ast::Expression {
location,
node: ast::ExpressionType::Compare { vals, ops }
}
},
Expression,
};
@@ -665,22 +704,34 @@ CompOp: ast::Comparison = {
};
Expression: ast::Expression = {
<e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
<e1:Expression> <location:@L> "|" <e2:XorExpression> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }
},
XorExpression,
};
XorExpression: ast::Expression = {
<e1:XorExpression> "^" <e2:AndExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) },
<e1:XorExpression> <location:@L> "^" <e2:AndExpression> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) }
},
AndExpression,
};
AndExpression: ast::Expression = {
<e1:AndExpression> "&" <e2:ShiftExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) },
<e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) }
},
ShiftExpression,
};
ShiftExpression: ast::Expression = {
<e1:ShiftExpression> <op:ShiftOp> <e2:ArithmaticExpression> => ast::Expression::Binop { a: Box::new(e1), op: op, b: Box::new(e2) },
<e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmaticExpression> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(e1), op: op, b: Box::new(e2) }
},
ArithmaticExpression,
};
@@ -690,7 +741,10 @@ ShiftOp: ast::Operator = {
};
ArithmaticExpression: ast::Expression = {
<a:ArithmaticExpression> <op:AddOp> <b:Term> => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) },
<a:ArithmaticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(a), op: op, b: Box::new(b) }
},
Term,
};
@@ -700,7 +754,10 @@ AddOp: ast::Operator = {
};
Term: ast::Expression = {
<a:Term> <op:MulOp> <b:Factor> => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) },
<a:Term> <location:@L> <op:MulOp> <b:Factor> => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) }
},
Factor,
};
@@ -713,25 +770,38 @@ MulOp: ast::Operator = {
};
Factor: ast::Expression = {
"+" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos },
"-" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg },
"~" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv },
<location:@L> <op:UnOp> <e:Factor> => ast::Expression {
location,
node: ast::ExpressionType::Unop { a: Box::new(e), op }
},
Power,
};
UnOp: ast::UnaryOperator = {
"+" => ast::UnaryOperator::Pos,
"-" => ast::UnaryOperator::Neg,
"~" => ast::UnaryOperator::Inv,
};
Power: ast::Expression = {
<e:AtomExpr> <e2:("**" Factor)?> => {
<e:AtomExpr> <e2:(@L "**" Factor)?> => {
match e2 {
None => e,
Some(x) => ast::Expression::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(x.1) },
Some((location, _, b)) => ast::Expression {
location,
node: ast::ExpressionType::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(b) }
},
}
}
};
AtomExpr: ast::Expression = {
<is_await:"await"?> <atom:AtomExpr2> => {
<location:@L> <is_await:"await"?> <atom:AtomExpr2> => {
if is_await.is_some() {
ast::Expression::Await { value: Box::new(atom) }
ast::Expression {
location,
node: ast::ExpressionType::Await { value: Box::new(atom) }
}
} else {
atom
}
@@ -740,13 +810,25 @@ AtomExpr: ast::Expression = {
AtomExpr2: ast::Expression = {
Atom,
<f:AtomExpr2> "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
<e:AtomExpr2> "[" <s:SubscriptList> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
<e:AtomExpr2> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
<f:AtomExpr2> <location:@L> "(" <a:ArgumentList> ")" => {
let (args, keywords) = a;
ast::Expression {
location,
node: ast::ExpressionType::Call { function: Box::new(f), args, keywords }
}
},
<e:AtomExpr2> <location:@L> "[" <s:SubscriptList> "]" => ast::Expression {
location,
node: ast::ExpressionType::Subscript { a: Box::new(e), b: Box::new(s) }
},
<e:AtomExpr2> <location:@L> "." <name:Identifier> => ast::Expression {
location,
node: ast::ExpressionType::Attribute { value: Box::new(e), name }
},
};
SubscriptList: ast::Expression = {
<s1:Subscript> <s2:("," Subscript)*> ","? => {
<location:@L> <s1:Subscript> <s2:("," Subscript)*> ","? => {
if s2.is_empty() {
s1
} else {
@@ -754,52 +836,86 @@ SubscriptList: ast::Expression = {
for x in s2 {
dims.push(x.1)
}
ast::Expression::Tuple { elements: dims }
ast::Expression {
location,
node: ast::ExpressionType::Tuple { elements: dims },
}
}
}
};
Subscript: ast::Expression = {
Test,
<e1:Test?> ":" <e2:Test?> <e3:SliceOp?> => {
let s1 = e1.unwrap_or(ast::Expression::None);
let s2 = e2.unwrap_or(ast::Expression::None);
let s3 = e3.unwrap_or(ast::Expression::None);
ast::Expression::Slice { elements: vec![s1, s2, s3] }
<e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> => {
let s1 = e1.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
let s2 = e2.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
let s3 = e3.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
ast::Expression {
location,
node: ast::ExpressionType::Slice { elements: vec![s1, s2, s3] }
}
}
};
SliceOp: ast::Expression = {
":" <e:Test?> => e.unwrap_or(ast::Expression::None)
<location:@L> ":" <e:Test?> => e.unwrap_or(ast::Expression {location, node: ast::ExpressionType::None})
}
Atom: ast::Expression = {
<value:StringGroup> => ast::Expression::String { value },
<value:Bytes> => ast::Expression::Bytes { value },
<value:Number> => ast::Expression::Number { value },
<name:Identifier> => ast::Expression::Identifier { name },
"[" <e:TestListComp?> "]" => {
<location:@L> <value:StringGroup> => ast::Expression {
location,
node: ast::ExpressionType::String { value }
},
<location:@L> <value:Bytes> => ast::Expression {
location,
node: ast::ExpressionType::Bytes { value }
},
<location:@L> <value:Number> => ast::Expression {
location,
node: ast::ExpressionType::Number { value }
},
<location:@L> <name:Identifier> => ast::Expression {
location,
node: ast::ExpressionType::Identifier { name }
},
<location:@L> "[" <e:TestListComp?> "]" => {
let elements = e.unwrap_or(Vec::new());
ast::Expression::List { elements }
},
"[" <e:TestListComp2> "]" => e,
"(" <elements:TestList?> ")" => {
elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() })
},
"(" <e:Test> <c:CompFor> ")" => {
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }),
generators: c,
ast::Expression {
location,
node: ast::ExpressionType::List { elements }
}
},
"{" <e:TestDict?> "}" => ast::Expression::Dict { elements: e.unwrap_or(Vec::new()) },
"[" <e:TestListComp2> "]" => e,
<location:@L> "(" <elements:TestList?> ")" => {
elements.unwrap_or(ast::Expression {
location,
node: ast::ExpressionType::Tuple { elements: Vec::new() }
})
},
<location:@L> "(" <e:Test> <c:CompFor> ")" => {
ast::Expression {
location,
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }),
generators: c,
}
}
},
<location:@L> "{" <e:TestDict?> "}" => ast::Expression {
location,
node: ast::ExpressionType::Dict { elements: e.unwrap_or(Vec::new()) }
},
"{" <e:TestDictComp> "}" => e,
"{" <e:TestSet> "}" => ast::Expression::Set { elements: e },
<location:@L> "{" <e:TestSet> "}" => ast::Expression {
location,
node: ast::ExpressionType::Set { elements: e }
},
"{" <e:TestSetComp> "}" => e,
"True" => ast::Expression::True,
"False" => ast::Expression::False,
"None" => ast::Expression::None,
"..." => ast::Expression::Ellipsis,
<location:@L> "True" => ast::Expression { location, node: ast::ExpressionType::True },
<location:@L> "False" => ast::Expression { location, node: ast::ExpressionType::False },
<location:@L> "None" => ast::Expression { location, node: ast::ExpressionType::None },
<location:@L> "..." => ast::Expression { location, node: ast::ExpressionType::Ellipsis },
};
TestListComp: Vec<ast::Expression> = {
@@ -807,10 +923,13 @@ TestListComp: Vec<ast::Expression> = {
};
TestListComp2: ast::Expression = {
<e:TestOrStarExpr> <c:CompFor> => {
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::List { element: e }),
generators: c,
<location:@L> <e:TestOrStarExpr> <c:CompFor> => {
ast::Expression {
location,
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::List { element: e }),
generators: c,
}
}
},
};
@@ -820,10 +939,13 @@ TestDict: Vec<(Option<ast::Expression>, ast::Expression)> = {
};
TestDictComp: ast::Expression = {
<e1:DictEntry> <c:CompFor> => {
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }),
generators: c,
<location:@L> <e1:DictEntry> <c:CompFor> => {
ast::Expression {
location,
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }),
generators: c,
}
}
}
};
@@ -842,10 +964,13 @@ TestSet: Vec<ast::Expression> = {
};
TestSetComp: ast::Expression = {
<e1:Test> <c:CompFor> => {
ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::Set { element: e1 }),
generators: c,
<location:@L> <e1:Test> <c:CompFor> => {
ast::Expression {
location,
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::Set { element: e1 }),
generators: c,
}
}
}
};
@@ -856,11 +981,14 @@ ExpressionOrStarExpression = {
};
ExpressionList: ast::Expression = {
<elements: OneOrMore<ExpressionOrStarExpression>> <trailing_comma:","?> => {
<location:@L> <elements: OneOrMore<ExpressionOrStarExpression>> <trailing_comma:","?> => {
if elements.len() == 1 && trailing_comma.is_none() {
elements.into_iter().next().unwrap()
} else {
ast::Expression::Tuple { elements }
ast::Expression {
location,
node: ast::ExpressionType::Tuple { elements },
}
}
},
};
@@ -874,18 +1002,24 @@ ExpressionList2: Vec<ast::Expression> = {
// - a single expression
// - a single expression followed by a trailing comma
TestList: ast::Expression = {
<elements:OneOrMore<Test>> <trailing_comma: ","?> => {
<location:@L> <elements:OneOrMore<Test>> <trailing_comma: ","?> => {
if elements.len() == 1 && trailing_comma.is_none() {
elements.into_iter().next().unwrap()
} else {
ast::Expression::Tuple { elements }
ast::Expression {
location,
node: ast::ExpressionType::Tuple { elements },
}
}
}
};
// Test
StarExpr: ast::Expression = {
"*" <e:Expression> => ast::Expression::Starred { value: Box::new(e) },
<location:@L> "*" <e:Expression> => ast::Expression {
location,
node: ast::ExpressionType::Starred { value: Box::new(e) },
}
};
// Comprehensions:
@@ -911,7 +1045,7 @@ ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
},
None => {
// Allow starred args after keyword arguments.
let is_starred = if let ast::Expression::Starred { .. } = &value {
let is_starred = if let ast::ExpressionType::Starred { .. } = &value.node {
true
} else {
false
@@ -931,16 +1065,19 @@ ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
FunctionArgument: (Option<Option<String>>, ast::Expression) = {
<e:Test> <c:CompFor?> => {
let expr = match c {
Some(c) => ast::Expression::Comprehension {
kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }),
generators: c,
Some(c) => ast::Expression {
location: e.location.clone(),
node: ast::ExpressionType::Comprehension {
kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }),
generators: c,
}
},
None => e,
};
(None, expr)
},
<i:Identifier> "=" <e:Test> => (Some(Some(i.clone())), e),
"*" <e:Test> => (None, ast::Expression::Starred { value: Box::new(e) }),
<location:@L> "*" <e:Test> => (None, ast::Expression { location, node: ast::ExpressionType::Starred { value: Box::new(e) } }),
"**" <e:Test> => (Some(None), e),
};
vm/src/stdlib/ast.rs
+61 -67
View File
@@ -58,10 +58,7 @@ fn create_node(vm: &VirtualMachine, name: &str) -> PyResult<AstNodeRef> {
AstNode.into_ref_with_type(vm, vm.class("ast", name))
}
fn statements_to_ast(
vm: &VirtualMachine,
statements: &[ast::LocatedStatement],
) -> PyResult<PyListRef> {
fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::Statement]) -> PyResult<PyListRef> {
let body: PyResult<Vec<_>> = statements
.iter()
.map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object()))
@@ -69,12 +66,10 @@ fn statements_to_ast(
Ok(vm.ctx.new_list(body?).downcast().unwrap())
}
fn statement_to_ast(
vm: &VirtualMachine,
statement: &ast::LocatedStatement,
) -> PyResult<AstNodeRef> {
fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult<AstNodeRef> {
use ast::StatementType::*;
let node = match &statement.node {
ast::Statement::ClassDef {
ClassDef {
name,
body,
keywords,
@@ -86,7 +81,7 @@ fn statement_to_ast(
body => statements_to_ast(vm, body)?,
decorator_list => expressions_to_ast(vm, decorator_list)?,
}),
ast::Statement::FunctionDef {
FunctionDef {
name,
args,
body,
@@ -99,7 +94,7 @@ fn statement_to_ast(
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
}),
ast::Statement::AsyncFunctionDef {
AsyncFunctionDef {
name,
args,
body,
@@ -112,14 +107,14 @@ fn statement_to_ast(
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
}),
ast::Statement::Continue => node!(vm, Continue),
ast::Statement::Break => node!(vm, Break),
ast::Statement::Pass => node!(vm, Pass),
ast::Statement::Assert { test, msg } => node!(vm, Assert, {
Continue => node!(vm, Continue),
Break => node!(vm, Break),
Pass => node!(vm, Pass),
Assert { test, msg } => node!(vm, Assert, {
test => expression_to_ast(vm, test)?,
msg => optional_expression_to_ast(vm, msg)?
}),
ast::Statement::Delete { targets } => {
Delete { targets } => {
let targets: PyResult<_> = targets
.iter()
.map(|v| Ok(expression_to_ast(vm, v)?.into_object()))
@@ -127,15 +122,15 @@ fn statement_to_ast(
let py_targets = vm.ctx.new_tuple(targets?);
node!(vm, Delete, { targets => py_targets })
}
ast::Statement::Return { value } => node!(vm, Return, {
Return { value } => node!(vm, Return, {
value => optional_expression_to_ast(vm, value)?
}),
ast::Statement::If { test, body, orelse } => node!(vm, If, {
If { test, body, orelse } => node!(vm, If, {
test => expression_to_ast(vm, test)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
ast::Statement::For {
For {
target,
iter,
body,
@@ -146,7 +141,7 @@ fn statement_to_ast(
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
ast::Statement::AsyncFor {
AsyncFor {
target,
iter,
body,
@@ -157,16 +152,16 @@ fn statement_to_ast(
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
ast::Statement::While { test, body, orelse } => node!(vm, While, {
While { test, body, orelse } => node!(vm, While, {
test => expression_to_ast(vm, test)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
ast::Statement::With { items, body } => node!(vm, With, {
With { items, body } => node!(vm, With, {
items => map_ast(with_item_to_ast, vm, items)?,
body => statements_to_ast(vm, body)?
}),
ast::Statement::Try {
Try {
body,
handlers,
orelse,
@@ -177,13 +172,13 @@ fn statement_to_ast(
orelse => optional_statements_to_ast(vm, orelse)?,
finalbody => optional_statements_to_ast(vm, finalbody)?
}),
ast::Statement::Expression { expression } => node!(vm, Expr, {
Expression { expression } => node!(vm, Expr, {
value => expression_to_ast(vm, expression)?
}),
ast::Statement::Import { names } => node!(vm, Import, {
Import { names } => node!(vm, Import, {
names => map_ast(alias_to_ast, vm, names)?
}),
ast::Statement::ImportFrom {
ImportFrom {
level,
module,
names,
@@ -192,22 +187,22 @@ fn statement_to_ast(
module => optional_string_to_py_obj(vm, module),
names => map_ast(alias_to_ast, vm, names)?
}),
ast::Statement::Nonlocal { names } => node!(vm, Nonlocal, {
Nonlocal { names } => node!(vm, Nonlocal, {
names => make_string_list(vm, names)
}),
ast::Statement::Global { names } => node!(vm, Global, {
Global { names } => node!(vm, Global, {
names => make_string_list(vm, names)
}),
ast::Statement::Assign { targets, value } => node!(vm, Assign, {
Assign { targets, value } => node!(vm, Assign, {
targets => expressions_to_ast(vm, targets)?,
value => expression_to_ast(vm, value)?,
}),
ast::Statement::AugAssign { target, op, value } => node!(vm, AugAssign, {
AugAssign { target, op, value } => node!(vm, AugAssign, {
target => expression_to_ast(vm, target)?,
op => vm.ctx.new_str(operator_string(op)),
value => expression_to_ast(vm, value)?,
}),
ast::Statement::Raise { exception, cause } => node!(vm, Raise, {
Raise { exception, cause } => node!(vm, Raise, {
exc => optional_expression_to_ast(vm, exception)?,
cause => optional_expression_to_ast(vm, cause)?,
}),
@@ -229,7 +224,7 @@ fn alias_to_ast(vm: &VirtualMachine, alias: &ast::ImportSymbol) -> PyResult<AstN
fn optional_statements_to_ast(
vm: &VirtualMachine,
statements: &Option<Vec<ast::LocatedStatement>>,
statements: &Option<Vec<ast::Statement>>,
) -> PyResult {
let statements = if let Some(statements) = statements {
statements_to_ast(vm, statements)?.into_object()
@@ -283,8 +278,9 @@ fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> P
}
fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult<AstNodeRef> {
let node = match &expression {
ast::Expression::Call {
use ast::ExpressionType::*;
let node = match &expression.node {
Call {
function,
args,
keywords,
@@ -293,7 +289,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
args => expressions_to_ast(vm, args)?,
keywords => map_ast(keyword_to_ast, vm, keywords)?,
}),
ast::Expression::Binop { a, op, b } => {
Binop { a, op, b } => {
// Operator:
node!(vm, BinOp, {
left => expression_to_ast(vm, a)?,
@@ -301,7 +297,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
right => expression_to_ast(vm, b)?,
})
}
ast::Expression::Unop { op, a } => {
Unop { op, a } => {
let op = match op {
ast::UnaryOperator::Not => "Not",
ast::UnaryOperator::Inv => "Invert",
@@ -313,7 +309,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
operand => expression_to_ast(vm, a)?,
})
}
ast::Expression::BoolOp { a, op, b } => {
BoolOp { a, op, b } => {
// Attach values:
let py_a = expression_to_ast(vm, a)?.into_object();
let py_b = expression_to_ast(vm, b)?.into_object();
@@ -330,7 +326,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
values => py_values,
})
}
ast::Expression::Compare { vals, ops } => {
Compare { vals, ops } => {
let left = expression_to_ast(vm, &vals[0])?;
// Operator:
@@ -364,20 +360,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
comparators => comparators,
})
}
ast::Expression::Identifier { name } => node!(vm, Name, {
Identifier { name } => node!(vm, Name, {
id => vm.ctx.new_str(name.clone()),
ctx => vm.ctx.none() // TODO: add context.
}),
ast::Expression::Lambda { args, body } => node!(vm, Lambda, {
Lambda { args, body } => node!(vm, Lambda, {
args => parameters_to_ast(vm, args)?,
body => expression_to_ast(vm, body)?,
}),
ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, {
IfExpression { test, body, orelse } => node!(vm, IfExp, {
text => expression_to_ast(vm, test)?,
body => expression_to_ast(vm, body)?,
or_else => expression_to_ast(vm, orelse)?,
}),
ast::Expression::Number { value } => {
Number { value } => {
let py_n = match value {
ast::Number::Integer { value } => vm.ctx.new_int(value.clone()),
ast::Number::Float { value } => vm.ctx.new_float(*value),
@@ -389,26 +385,26 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
n => py_n
})
}
ast::Expression::True => node!(vm, NameConstant, {
True => node!(vm, NameConstant, {
value => vm.ctx.new_bool(true)
}),
ast::Expression::False => node!(vm, NameConstant, {
False => node!(vm, NameConstant, {
value => vm.ctx.new_bool(false)
}),
ast::Expression::None => node!(vm, NameConstant, {
None => node!(vm, NameConstant, {
value => vm.ctx.none()
}),
ast::Expression::Ellipsis => node!(vm, Ellipsis),
ast::Expression::List { elements } => node!(vm, List, {
Ellipsis => node!(vm, Ellipsis),
List { elements } => node!(vm, List, {
elts => expressions_to_ast(vm, &elements)?
}),
ast::Expression::Tuple { elements } => node!(vm, Tuple, {
Tuple { elements } => node!(vm, Tuple, {
elts => expressions_to_ast(vm, &elements)?
}),
ast::Expression::Set { elements } => node!(vm, Set, {
Set { elements } => node!(vm, Set, {
elts => expressions_to_ast(vm, &elements)?
}),
ast::Expression::Dict { elements } => {
Dict { elements } => {
let mut keys = Vec::new();
let mut values = Vec::new();
for (k, v) in elements {
@@ -425,7 +421,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
values => vm.ctx.new_list(values),
})
}
ast::Expression::Comprehension { kind, generators } => {
Comprehension { kind, generators } => {
let py_generators = map_ast(comprehension_to_ast, vm, generators)?;
match kind.deref() {
@@ -443,50 +439,48 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
}
}
}
ast::Expression::Await { value } => {
Await { value } => {
let py_value = expression_to_ast(vm, value)?;
node!(vm, Await, {
value => py_value
})
}
ast::Expression::Yield { value } => {
let py_value = match value {
Some(value) => expression_to_ast(vm, value)?.into_object(),
None => vm.ctx.none(),
Yield { value } => {
let py_value = if let Some(value) = value {
expression_to_ast(vm, value)?.into_object()
} else {
vm.ctx.none()
};
node!(vm, Yield, {
value => py_value
})
}
ast::Expression::YieldFrom { value } => {
YieldFrom { value } => {
let py_value = expression_to_ast(vm, value)?;
node!(vm, YieldFrom, {
value => py_value
})
}
ast::Expression::Subscript { a, b } => node!(vm, Subscript, {
Subscript { a, b } => node!(vm, Subscript, {
value => expression_to_ast(vm, a)?,
slice => expression_to_ast(vm, b)?,
}),
ast::Expression::Attribute { value, name } => node!(vm, Attribute, {
Attribute { value, name } => node!(vm, Attribute, {
value => expression_to_ast(vm, value)?,
attr => vm.ctx.new_str(name.to_string()),
ctx => vm.ctx.none()
}),
ast::Expression::Starred { value } => node!(vm, Starred, {
Starred { value } => node!(vm, Starred, {
value => expression_to_ast(vm, value)?
}),
ast::Expression::Slice { elements } => node!(vm, Slice, {
Slice { elements } => node!(vm, Slice, {
bounds => expressions_to_ast(vm, elements)?
}),
ast::Expression::String { value } => string_to_ast(vm, value)?,
ast::Expression::Bytes { value } => {
node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) })
}
String { value } => string_to_ast(vm, value)?,
Bytes { value } => node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }),
};
// TODO: retrieve correct lineno:
let lineno = vm.ctx.new_int(1);
let lineno = vm.ctx.new_int(expression.location.row());
vm.set_attr(node.as_object(), "lineno", lineno).unwrap();
Ok(node)
}
vm/src/stdlib/mod.rs
+6 -1
View File
@@ -61,7 +61,6 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
"random".to_string() => Box::new(random::make_module),
"_string".to_string() => Box::new(string::make_module),
"struct".to_string() => Box::new(pystruct::make_module),
"symtable".to_string() => Box::new(symtable::make_module),
"_thread".to_string() => Box::new(thread::make_module),
"time".to_string() => Box::new(time_module::make_module),
"_weakref".to_string() => Box::new(weakref::make_module),
@@ -81,6 +80,12 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
modules.insert("tokenize".to_string(), Box::new(tokenize::make_module));
}
// Insert compiler related modules:
#[cfg(feature = "rustpython-compiler")]
{
modules.insert("symtable".to_string(), Box::new(symtable::make_module));
}
// disable some modules on WASM
#[cfg(not(target_arch = "wasm32"))]
{
vm/src/stdlib/symtable.rs
+16 -44
View File
@@ -89,11 +89,9 @@ impl PySymbolTable {
#[pymethod(name = "lookup")]
fn lookup(&self, name: PyStringRef, vm: &VirtualMachine) -> PyResult<PySymbolRef> {
let name = &name.value;
if self.symtable.symbols.contains_key(name) {
let role = self.symtable.symbols.get(name).unwrap().clone();
if let Some(symbol) = self.symtable.symbols.get(name) {
Ok(PySymbol {
name: name.to_string(),
role,
symbol: symbol.clone(),
}
.into_ref(vm))
} else {
@@ -106,15 +104,8 @@ impl PySymbolTable {
let symbols = self
.symtable
.symbols
.iter()
.map(|s| {
(PySymbol {
name: s.0.clone(),
role: s.1.clone(),
})
.into_ref(vm)
.into_object()
})
.values()
.map(|s| (PySymbol { symbol: s.clone() }).into_ref(vm).into_object())
.collect();
Ok(vm.ctx.new_list(symbols))
}
@@ -137,8 +128,7 @@ impl PySymbolTable {
#[pyclass(name = "Symbol")]
struct PySymbol {
name: String,
role: symboltable::SymbolRole,
symbol: symboltable::Symbol,
}
impl fmt::Debug for PySymbol {
@@ -157,55 +147,37 @@ impl PyValue for PySymbol {
impl PySymbol {
#[pymethod(name = "get_name")]
fn get_name(&self, vm: &VirtualMachine) -> PyResult {
Ok(vm.ctx.new_str(self.name.clone()))
Ok(vm.ctx.new_str(self.symbol.name.clone()))
}
#[pymethod(name = "is_global")]
fn is_global(&self, vm: &VirtualMachine) -> PyResult {
// TODO: figure out how this is determined.
let is_local = if let symboltable::SymbolRole::Used = self.role {
true
} else {
false
};
Ok(vm.ctx.new_bool(is_local))
Ok(vm.ctx.new_bool(self.symbol.is_global))
}
#[pymethod(name = "is_local")]
fn is_local(&self, vm: &VirtualMachine) -> PyResult {
// TODO: figure out how this is determined.
let is_local = if let symboltable::SymbolRole::Used = self.role {
true
} else {
false
};
Ok(vm.ctx.new_bool(is_local))
Ok(vm.ctx.new_bool(self.symbol.is_local))
}
#[pymethod(name = "is_referenced")]
fn is_referenced(&self, vm: &VirtualMachine) -> PyResult {
let is_local = if let symboltable::SymbolRole::Used = self.role {
true
} else {
false
};
Ok(vm.ctx.new_bool(is_local))
Ok(vm.ctx.new_bool(self.symbol.is_referenced))
}
#[pymethod(name = "is_assigned")]
fn is_assigned(&self, vm: &VirtualMachine) -> PyResult {
let is_local = if let symboltable::SymbolRole::Assigned = self.role {
true
} else {
false
};
Ok(vm.ctx.new_bool(is_local))
Ok(vm.ctx.new_bool(self.symbol.is_assigned))
}
#[pymethod(name = "is_parameter")]
fn is_parameter(&self, vm: &VirtualMachine) -> PyResult {
// TODO
Ok(vm.ctx.new_bool(false))
Ok(vm.ctx.new_bool(self.symbol.is_parameter))
}
#[pymethod(name = "is_free")]
fn is_free(&self, vm: &VirtualMachine) -> PyResult {
Ok(vm.ctx.new_bool(self.symbol.is_free))
}
#[pymethod(name = "is_namespace")]