From 215308d474d21a8d55f19717ff5ae220b9ccdaff Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Tue, 16 Jul 2019 22:04:38 +0200 Subject: [PATCH] Add initial draft of symtable module. --- compiler/src/lib.rs | 2 +- compiler/src/symboltable.rs | 3 +- crawl_sourcecode.py | 12 +++ vm/src/builtins.rs | 16 +-- vm/src/eval.rs | 12 +++ vm/src/stdlib/ast.rs | 132 +++++++++++------------ vm/src/stdlib/mod.rs | 3 + vm/src/stdlib/symtable.rs | 209 ++++++++++++++++++++++++++++++++++++ 8 files changed, 307 insertions(+), 82 deletions(-) create mode 100644 vm/src/stdlib/symtable.rs diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 5e844b2e3..de949be2c 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -8,4 +8,4 @@ extern crate log; pub mod compile; pub mod error; -mod symboltable; +pub mod symboltable; diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 5490bd058..233010164 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -36,7 +36,7 @@ pub fn statements_to_symbol_table( Ok(symbol_table) } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum SymbolRole { Global, Nonlocal, @@ -45,6 +45,7 @@ pub enum SymbolRole { } /// Captures all symbols in the current scope, and has a list of subscopes in this scope. +#[derive(Clone)] pub struct SymbolScope { /// A set of symbols present on this scope level. pub symbols: HashMap, diff --git a/crawl_sourcecode.py b/crawl_sourcecode.py index 81cbff8af..7da570315 100644 --- a/crawl_sourcecode.py +++ b/crawl_sourcecode.py @@ -1,6 +1,7 @@ import ast import sys +import symtable filename = sys.argv[1] print('Crawling file:', filename) @@ -31,3 +32,14 @@ print_node(t) # print(ast.dump(t)) +def print_table(table, indent=0): + print(' '*indent, 'table:', table.get_name()) + print(' '*indent, ' ', 'Syms:') + for sym in table.get_symbols(): + print(' '*indent, ' ', sym) + print(' '*indent, ' ', 'Child tables:') + for child in table.get_children(): + print_table(child, indent=indent+shift) + +table = symtable.symtable(source, 'a', 'exec') +print_table(table) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 55f85baaa..17abf16b4 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -21,6 +21,7 @@ use crate::obj::objtype::{self, PyClassRef}; #[cfg(feature = "rustpython-compiler")] use rustpython_compiler::compile; +use crate::eval::get_compile_mode; use crate::frame::Scope; use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ @@ -110,20 +111,7 @@ fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult PyResult { + if mode == "exec" { + Ok(compile::Mode::Exec) + } else if mode == "eval" { + Ok(compile::Mode::Eval) + } else if mode == "single" { + Ok(compile::Mode::Single) + } else { + Err(vm.new_value_error("compile() mode must be 'exec', 'eval' or single'".to_string())) + } +} + #[cfg(test)] mod tests { use super::eval; diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 1fae21741..c335a7f0d 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -364,8 +364,9 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes comparators => comparators, }) } - ast::Expression::Identifier { name } => node!(vm, Identifier, { - id => vm.ctx.new_str(name.clone()) + ast::Expression::Identifier { name } => node!(vm, Name, { + id => vm.ctx.new_str(name.clone()), + ctx => vm.ctx.none() // TODO: add context. }), ast::Expression::Lambda { args, body } => node!(vm, Lambda, { args => parameters_to_ast(vm, args)?, @@ -470,6 +471,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes ast::Expression::Attribute { value, name } => node!(vm, Attribute, { value => expression_to_ast(vm, value)?, attr => vm.ctx.new_str(name.to_string()), + ctx => vm.ctx.none() }), ast::Expression::Starred { value } => node!(vm, Starred, { value => expression_to_ast(vm, value)? @@ -589,73 +591,71 @@ fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); + let ast_base = py_class!(ctx, "AST", ctx.object(), {}); py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), "AST" => ast_base.clone(), // TODO: There's got to be a better way! - "alias" => py_class!(ctx, "_ast.alias", ast_base.clone(), {}), - "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), - "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), - "Assign" => py_class!(ctx, "_ast.Assign", ast_base.clone(), {}), - "AugAssign" => py_class!(ctx, "_ast.AugAssign", ast_base.clone(), {}), - "AsyncFor" => py_class!(ctx, "_ast.AsyncFor", ast_base.clone(), {}), - "AsyncFunctionDef" => py_class!(ctx, "_ast.AsyncFunctionDef", ast_base.clone(), {}), - "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), - "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), - "Await" => py_class!(ctx, "_ast.Await", ast_base.clone(), {}), - "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), - "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), - "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), - "Bytes" => py_class!(ctx, "_ast.Bytes", ast_base.clone(), {}), - "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), - "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), - "Compare" => py_class!(ctx, "_ast.Compare", ast_base.clone(), {}), - "comprehension" => py_class!(ctx, "_ast.comprehension", ast_base.clone(), {}), - "Continue" => py_class!(ctx, "_ast.Continue", ast_base.clone(), {}), - "Delete" => py_class!(ctx, "_ast.Delete", ast_base.clone(), {}), - "Dict" => py_class!(ctx, "_ast.Dict", ast_base.clone(), {}), - "DictComp" => py_class!(ctx, "_ast.DictComp", ast_base.clone(), {}), - "Ellipsis" => py_class!(ctx, "_ast.Ellipsis", ast_base.clone(), {}), - "Expr" => py_class!(ctx, "_ast.Expr", ast_base.clone(), {}), - "ExceptHandler" => py_class!(ctx, "_ast.ExceptHandler", ast_base.clone(), {}), - "For" => py_class!(ctx, "_ast.For", ast_base.clone(), {}), - "FormattedValue" => py_class!(ctx, "_ast.FormattedValue", ast_base.clone(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), - "GeneratorExp" => py_class!(ctx, "_ast.GeneratorExp", ast_base.clone(), {}), - "Global" => py_class!(ctx, "_ast.Global", ast_base.clone(), {}), - "Identifier" => py_class!(ctx, "_ast.Identifier", ast_base.clone(), {}), - "If" => py_class!(ctx, "_ast.If", ast_base.clone(), {}), - "IfExp" => py_class!(ctx, "_ast.IfExp", ast_base.clone(), {}), - "Import" => py_class!(ctx, "_ast.Import", ast_base.clone(), {}), - "ImportFrom" => py_class!(ctx, "_ast.ImportFrom", ast_base.clone(), {}), - "JoinedStr" => py_class!(ctx, "_ast.JoinedStr", ast_base.clone(), {}), - "keyword" => py_class!(ctx, "_ast.keyword", ast_base.clone(), {}), - "Lambda" => py_class!(ctx, "_ast.Lambda", ast_base.clone(), {}), - "List" => py_class!(ctx, "_ast.List", ast_base.clone(), {}), - "ListComp" => py_class!(ctx, "_ast.ListComp", ast_base.clone(), {}), - "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), - "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), - "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), - "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), - "Nonlocal" => py_class!(ctx, "_ast.Nonlocal", ast_base.clone(), {}), - "Num" => py_class!(ctx, "_ast.Num", ast_base.clone(), {}), - "Pass" => py_class!(ctx, "_ast.Pass", ast_base.clone(), {}), - "Raise" => py_class!(ctx, "_ast.Raise", ast_base.clone(), {}), - "Return" => py_class!(ctx, "_ast.Return", ast_base.clone(), {}), - "Set" => py_class!(ctx, "_ast.Set", ast_base.clone(), {}), - "SetComp" => py_class!(ctx, "_ast.SetComp", ast_base.clone(), {}), - "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), - "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), - "Str" => py_class!(ctx, "_ast.Str", ast_base.clone(), {}), - "Subscript" => py_class!(ctx, "_ast.Subscript", ast_base.clone(), {}), - "Try" => py_class!(ctx, "_ast.Try", ast_base.clone(), {}), - "Tuple" => py_class!(ctx, "_ast.Tuple", ast_base.clone(), {}), - "UnaryOp" => py_class!(ctx, "_ast.UnaryOp", ast_base.clone(), {}), - "While" => py_class!(ctx, "_ast.While", ast_base.clone(), {}), - "With" => py_class!(ctx, "_ast.With", ast_base.clone(), {}), - "withitem" => py_class!(ctx, "_ast.withitem", ast_base.clone(), {}), - "Yield" => py_class!(ctx, "_ast.Yield", ast_base.clone(), {}), - "YieldFrom" => py_class!(ctx, "_ast.YieldFrom", ast_base.clone(), {}), + "alias" => py_class!(ctx, "alias", ast_base.clone(), {}), + "arg" => py_class!(ctx, "arg", ast_base.clone(), {}), + "arguments" => py_class!(ctx, "arguments", ast_base.clone(), {}), + "Assign" => py_class!(ctx, "Assign", ast_base.clone(), {}), + "AugAssign" => py_class!(ctx, "AugAssign", ast_base.clone(), {}), + "AsyncFor" => py_class!(ctx, "AsyncFor", ast_base.clone(), {}), + "AsyncFunctionDef" => py_class!(ctx, "AsyncFunctionDef", ast_base.clone(), {}), + "Assert" => py_class!(ctx, "Assert", ast_base.clone(), {}), + "Attribute" => py_class!(ctx, "Attribute", ast_base.clone(), {}), + "Await" => py_class!(ctx, "Await", ast_base.clone(), {}), + "BinOp" => py_class!(ctx, "BinOp", ast_base.clone(), {}), + "BoolOp" => py_class!(ctx, "BoolOp", ast_base.clone(), {}), + "Break" => py_class!(ctx, "Break", ast_base.clone(), {}), + "Bytes" => py_class!(ctx, "Bytes", ast_base.clone(), {}), + "Call" => py_class!(ctx, "Call", ast_base.clone(), {}), + "ClassDef" => py_class!(ctx, "ClassDef", ast_base.clone(), {}), + "Compare" => py_class!(ctx, "Compare", ast_base.clone(), {}), + "comprehension" => py_class!(ctx, "comprehension", ast_base.clone(), {}), + "Continue" => py_class!(ctx, "Continue", ast_base.clone(), {}), + "Delete" => py_class!(ctx, "Delete", ast_base.clone(), {}), + "Dict" => py_class!(ctx, "Dict", ast_base.clone(), {}), + "DictComp" => py_class!(ctx, "DictComp", ast_base.clone(), {}), + "Ellipsis" => py_class!(ctx, "Ellipsis", ast_base.clone(), {}), + "Expr" => py_class!(ctx, "Expr", ast_base.clone(), {}), + "ExceptHandler" => py_class!(ctx, "ExceptHandler", ast_base.clone(), {}), + "For" => py_class!(ctx, "For", ast_base.clone(), {}), + "FormattedValue" => py_class!(ctx, "FormattedValue", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "FunctionDef", ast_base.clone(), {}), + "GeneratorExp" => py_class!(ctx, "GeneratorExp", ast_base.clone(), {}), + "Global" => py_class!(ctx, "Global", ast_base.clone(), {}), + "If" => py_class!(ctx, "If", ast_base.clone(), {}), + "IfExp" => py_class!(ctx, "IfExp", ast_base.clone(), {}), + "Import" => py_class!(ctx, "Import", ast_base.clone(), {}), + "ImportFrom" => py_class!(ctx, "ImportFrom", ast_base.clone(), {}), + "JoinedStr" => py_class!(ctx, "JoinedStr", ast_base.clone(), {}), + "keyword" => py_class!(ctx, "keyword", ast_base.clone(), {}), + "Lambda" => py_class!(ctx, "Lambda", ast_base.clone(), {}), + "List" => py_class!(ctx, "List", ast_base.clone(), {}), + "ListComp" => py_class!(ctx, "ListComp", ast_base.clone(), {}), + "Module" => py_class!(ctx, "Module", ast_base.clone(), {}), + "Name" => py_class!(ctx, "Name", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "NameConstant", ast_base.clone(), {}), + "Nonlocal" => py_class!(ctx, "Nonlocal", ast_base.clone(), {}), + "Num" => py_class!(ctx, "Num", ast_base.clone(), {}), + "Pass" => py_class!(ctx, "Pass", ast_base.clone(), {}), + "Raise" => py_class!(ctx, "Raise", ast_base.clone(), {}), + "Return" => py_class!(ctx, "Return", ast_base.clone(), {}), + "Set" => py_class!(ctx, "Set", ast_base.clone(), {}), + "SetComp" => py_class!(ctx, "SetComp", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "Starred", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "Starred", ast_base.clone(), {}), + "Str" => py_class!(ctx, "Str", ast_base.clone(), {}), + "Subscript" => py_class!(ctx, "Subscript", ast_base.clone(), {}), + "Try" => py_class!(ctx, "Try", ast_base.clone(), {}), + "Tuple" => py_class!(ctx, "Tuple", ast_base.clone(), {}), + "UnaryOp" => py_class!(ctx, "UnaryOp", ast_base.clone(), {}), + "While" => py_class!(ctx, "While", ast_base.clone(), {}), + "With" => py_class!(ctx, "With", ast_base.clone(), {}), + "withitem" => py_class!(ctx, "withitem", ast_base.clone(), {}), + "Yield" => py_class!(ctx, "Yield", ast_base.clone(), {}), + "YieldFrom" => py_class!(ctx, "YieldFrom", ast_base.clone(), {}), }) } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index a0499002e..b189ad8db 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -19,6 +19,8 @@ mod random; mod re; pub mod socket; mod string; +#[cfg(feature = "rustpython-compiler")] +mod symtable; mod thread; mod time_module; #[cfg(feature = "rustpython-parser")] @@ -59,6 +61,7 @@ pub fn get_module_inits() -> HashMap { "random".to_string() => Box::new(random::make_module), "_string".to_string() => Box::new(string::make_module), "struct".to_string() => Box::new(pystruct::make_module), + "symtable".to_string() => Box::new(symtable::make_module), "_thread".to_string() => Box::new(thread::make_module), "time".to_string() => Box::new(time_module::make_module), "_weakref".to_string() => Box::new(weakref::make_module), diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs new file mode 100644 index 000000000..6387dd769 --- /dev/null +++ b/vm/src/stdlib/symtable.rs @@ -0,0 +1,209 @@ +use crate::eval::get_compile_mode; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; +use rustpython_compiler::{compile, error::CompileError, symboltable}; +use rustpython_parser::parser; +use std::fmt; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let symbol_table_type = PySymbolTable::make_class(ctx); + let symbol_type = PySymbol::make_class(ctx); + + py_module!(vm, "symtable", { + "symtable" => ctx.new_rustfunc(symtable_symtable), + "SymbolTable" => symbol_table_type, + "Symbol" => symbol_type, + }) +} + +/// symtable. Return top level SymbolTable. +/// See docs: https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.symtable +fn symtable_symtable( + source: PyStringRef, + _filename: PyStringRef, + mode: PyStringRef, + vm: &VirtualMachine, +) -> PyResult { + let mode = get_compile_mode(vm, &mode.value)?; + let symtable = + source_to_symtable(&source.value, mode).map_err(|err| vm.new_syntax_error(&err))?; + + let py_symbol_table = to_py_symbol_table("top".to_string(), symtable); + Ok(py_symbol_table.into_ref(vm)) +} + +fn source_to_symtable( + source: &str, + mode: compile::Mode, +) -> Result { + let symtable = match mode { + compile::Mode::Exec | compile::Mode::Single => { + let ast = parser::parse_program(source)?; + symboltable::make_symbol_table(&ast)? + } + compile::Mode::Eval => { + let statement = parser::parse_statement(source)?; + symboltable::statements_to_symbol_table(&statement)? + } + }; + + Ok(symtable) +} + +fn to_py_symbol_table(name: String, symtable: symboltable::SymbolScope) -> PySymbolTable { + PySymbolTable { name, symtable } +} + +type PySymbolTableRef = PyRef; +type PySymbolRef = PyRef; + +#[pyclass(name = "SymbolTable")] +struct PySymbolTable { + name: String, + symtable: symboltable::SymbolScope, +} + +impl fmt::Debug for PySymbolTable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "SymbolTable()") + } +} + +impl PyValue for PySymbolTable { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("symtable", "SymbolTable") + } +} + +#[pyimpl] +impl PySymbolTable { + #[pymethod(name = "get_name")] + fn get_name(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.name.clone())) + } + + #[pymethod(name = "lookup")] + fn lookup(&self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { + let name = &name.value; + if self.symtable.symbols.contains_key(name) { + let role = self.symtable.symbols.get(name).unwrap().clone(); + Ok(PySymbol { + name: name.to_string(), + role, + } + .into_ref(vm)) + } else { + Err(vm.ctx.new_str(name.to_string())) + } + } + + #[pymethod(name = "get_symbols")] + fn get_symbols(&self, vm: &VirtualMachine) -> PyResult { + let symbols = self + .symtable + .symbols + .iter() + .map(|s| vm.ctx.new_str(s.0.clone())) + .collect(); + Ok(vm.ctx.new_list(symbols)) + } + + #[pymethod(name = "get_children")] + fn get_children(&self, vm: &VirtualMachine) -> PyResult { + let children = self + .symtable + .sub_scopes + .iter() + .map(|s| { + to_py_symbol_table("bla".to_string(), s.clone()) + .into_ref(vm) + .into_object() + }) + .collect(); + Ok(vm.ctx.new_list(children)) + } +} + +#[pyclass(name = "Symbol")] +struct PySymbol { + name: String, + role: symboltable::SymbolRole, +} + +impl fmt::Debug for PySymbol { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Symbol()") + } +} + +impl PyValue for PySymbol { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("symtable", "Symbol") + } +} + +#[pyimpl] +impl PySymbol { + #[pymethod(name = "get_name")] + fn get_name(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.name.clone())) + } + + #[pymethod(name = "is_global")] + fn is_global(&self, vm: &VirtualMachine) -> PyResult { + // TODO: figure out how this is determined. + let is_local = if let symboltable::SymbolRole::Used = self.role { + true + } else { + false + }; + Ok(vm.ctx.new_bool(is_local)) + } + + #[pymethod(name = "is_local")] + fn is_local(&self, vm: &VirtualMachine) -> PyResult { + // TODO: figure out how this is determined. + let is_local = if let symboltable::SymbolRole::Used = self.role { + true + } else { + false + }; + Ok(vm.ctx.new_bool(is_local)) + } + + #[pymethod(name = "is_referenced")] + fn is_referenced(&self, vm: &VirtualMachine) -> PyResult { + let is_local = if let symboltable::SymbolRole::Used = self.role { + true + } else { + false + }; + Ok(vm.ctx.new_bool(is_local)) + } + + #[pymethod(name = "is_assigned")] + fn is_assigned(&self, vm: &VirtualMachine) -> PyResult { + let is_local = if let symboltable::SymbolRole::Assigned = self.role { + true + } else { + false + }; + Ok(vm.ctx.new_bool(is_local)) + } + + #[pymethod(name = "is_parameter")] + fn is_parameter(&self, vm: &VirtualMachine) -> PyResult { + // TODO + Ok(vm.ctx.new_bool(false)) + } + + #[pymethod(name = "is_namespace")] + fn is_namespace(&self, vm: &VirtualMachine) -> PyResult { + // TODO + Ok(vm.ctx.new_bool(false)) + } +}