diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 01e7fe1d6..9fd818536 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,11 +2,16 @@ ## Code organization +- `bytecode/src`: python bytecode representation in rust structures +- `compiler/src`: python compilation to bytecode - `parser/src`: python lexing, parsing and ast +- `Lib`: Carefully selected / copied files from CPython sourcecode. This is + the python side of the standard library. - `vm/src`: python virtual machine - `builtins.rs`: Builtin functions - `compile.rs`: the python compiler from ast to bytecode - `obj`: python builtin types + - `stdlib`: Standard library parts implemented in rust. - `src`: using the other subcrates to bring rustpython to life. - `docs`: documentation (work in progress) - `py_code_object`: CPython bytecode to rustpython bytecode converter (work in @@ -28,8 +33,7 @@ To test rustpython, there is a collection of python snippets located in the ```shell $ cd tests -$ pipenv install -$ pipenv run pytest -v +$ pytest -v ``` There also are some unit tests, you can run those with cargo: diff --git a/README.md b/README.md index 70b8f24dc..009135660 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ Documentation HTML files can then be found in the `target/doc` directory. ## Contributing Contributions are more than welcome, and in many cases we are happy to guide -contributors through PRs or on gitter. +contributors through PRs or on gitter. Please refer to the +[development guide](DEVELOPMENT.md) as well for tips on developments. With that in mind, please note this project is maintained by volunteers, some of the best ways to get started are below: diff --git a/src/main.rs b/src/main.rs index d4a0f2c4a..63af32d61 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,11 +8,11 @@ use clap::{App, Arg, ArgMatches}; use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType}; use rustpython_parser::error::ParseErrorType; use rustpython_vm::{ - frame::Scope, import, obj::objstr, print_exception, pyobject::{ItemProtocol, PyResult}, + scope::Scope, util, PySettings, VirtualMachine, }; use std::convert::TryInto; diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index ec71ac5fa..f422688e8 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -22,12 +22,12 @@ use crate::obj::objtype::{self, PyClassRef}; use rustpython_compiler::compile; use crate::eval::get_compile_mode; -use crate::frame::Scope; use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ Either, IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; +use crate::scope::Scope; use crate::vm::VirtualMachine; use crate::obj::objbyteinner::PyByteInner; diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 40f084df7..48106cd5c 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,5 +1,5 @@ -use crate::frame::Scope; use crate::pyobject::PyResult; +use crate::scope::Scope; use crate::vm::VirtualMachine; use rustpython_compiler::compile; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 93b2ff3cd..f3d6f0bed 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1,6 +1,5 @@ use std::cell::RefCell; use std::fmt; -use std::rc::Rc; use crate::builtins; use crate::bytecode; @@ -17,188 +16,13 @@ use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, + IdProtocol, ItemProtocol, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; +use crate::scope::{NameProtocol, Scope}; use crate::vm::VirtualMachine; use indexmap::IndexMap; use itertools::Itertools; -/* - * So a scope is a linked list of scopes. - * When a name is looked up, it is check in its scope. - */ -#[derive(Debug)] -struct RcListNode { - elem: T, - next: Option>>, -} - -#[derive(Debug, Clone)] -struct RcList { - head: Option>>, -} - -struct Iter<'a, T: 'a> { - next: Option<&'a RcListNode>, -} - -impl RcList { - pub fn new() -> Self { - RcList { head: None } - } - - pub fn insert(self, elem: T) -> Self { - RcList { - head: Some(Rc::new(RcListNode { - elem, - next: self.head, - })), - } - } - - #[cfg_attr(feature = "flame-it", flame("RcList"))] - pub fn iter(&self) -> Iter { - Iter { - next: self.head.as_ref().map(|node| &**node), - } - } -} - -impl<'a, T> Iterator for Iter<'a, T> { - type Item = &'a T; - - #[cfg_attr(feature = "flame-it", flame("Iter"))] - fn next(&mut self) -> Option { - self.next.map(|node| { - self.next = node.next.as_ref().map(|node| &**node); - &node.elem - }) - } -} - -#[derive(Clone)] -pub struct Scope { - locals: RcList, - pub globals: PyDictRef, -} - -impl fmt::Debug for Scope { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow - f.write_str("Scope") - } -} - -impl Scope { - pub fn new(locals: Option, globals: PyDictRef) -> Scope { - let locals = match locals { - Some(dict) => RcList::new().insert(dict), - None => RcList::new(), - }; - Scope { locals, globals } - } - - pub fn with_builtins( - locals: Option, - globals: PyDictRef, - vm: &VirtualMachine, - ) -> Scope { - if !globals.contains_key("__builtins__", vm) { - globals - .clone() - .set_item("__builtins__", vm.builtins.clone(), vm) - .unwrap(); - } - Scope::new(locals, globals) - } - - pub fn get_locals(&self) -> PyDictRef { - match self.locals.iter().next() { - Some(dict) => dict.clone(), - None => self.globals.clone(), - } - } - - pub fn get_only_locals(&self) -> Option { - self.locals.iter().next().cloned() - } - - pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope { - Scope { - locals: self.locals.clone().insert(locals), - globals: self.globals.clone(), - } - } - - pub fn new_child_scope(&self, ctx: &PyContext) -> Scope { - self.new_child_scope_with_locals(ctx.new_dict()) - } -} - -pub trait NameProtocol { - fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option; - fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); - fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult; - fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; - fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); - fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; - fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); -} - -impl NameProtocol for Scope { - #[cfg_attr(feature = "flame-it", flame("Scope"))] - fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { - for dict in self.locals.iter() { - if let Some(value) = dict.get_item_option(name, vm).unwrap() { - return Some(value); - } - } - - if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { - return Some(value); - } - - vm.get_attribute(vm.builtins.clone(), name).ok() - } - - #[cfg_attr(feature = "flame-it", flame("Scope"))] - fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { - for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item_option(name, vm).unwrap() { - return Some(value); - } - } - None - } - - fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { - self.locals - .iter() - .nth(1) - .expect("no outer scope for non-local") - .set_item(name, value, vm) - .unwrap(); - } - - fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(key, value, vm).unwrap(); - } - - fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult { - self.get_locals().del_item(key, vm) - } - - #[cfg_attr(feature = "flame-it", flame("Scope"))] - fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { - self.globals.get_item_option(name, vm).unwrap() - } - - fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { - self.globals.set_item(name, value, vm).unwrap(); - } -} - #[derive(Clone, Debug)] struct Block { /// The type of block. diff --git a/vm/src/import.rs b/vm/src/import.rs index 88d66ce82..e61c87640 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -3,9 +3,9 @@ */ use crate::bytecode::CodeObject; -use crate::frame::Scope; use crate::obj::{objcode, objsequence, objstr, objtype}; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; +use crate::scope::Scope; use crate::vm::VirtualMachine; #[cfg(feature = "rustpython-compiler")] use rustpython_compiler::compile; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 60dfa725e..c4f042858 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -56,7 +56,7 @@ mod dictdatatype; pub mod eval; mod exceptions; pub mod format; -pub mod frame; +mod frame; mod frozen; pub mod function; pub mod import; @@ -64,6 +64,7 @@ pub mod obj; pub mod py_serde; mod pyhash; pub mod pyobject; +pub mod scope; pub mod stdlib; mod sysmodule; mod traceback; diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 05b66c346..1c7ff642a 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,10 +1,10 @@ -use crate::frame::Scope; use crate::function::{Args, KwArgs}; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::scope::Scope; use crate::vm::VirtualMachine; pub type PyFunctionRef = PyRef; diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 194212a49..016c2419c 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -6,7 +6,6 @@ https://github.com/python/cpython/blob/50b48572d9a90c5bb36e2bef6179548ea927a35a/ */ -use crate::frame::NameProtocol; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objfunction::PyMethod; use crate::obj::objstr; @@ -14,6 +13,7 @@ use crate::obj::objtype::{PyClass, PyClassRef}; use crate::pyobject::{ ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; +use crate::scope::NameProtocol; use crate::vm::VirtualMachine; use super::objtype; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 3d3463a4d..bc9891511 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -15,7 +15,6 @@ use num_traits::{One, Zero}; use crate::bytecode; use crate::exceptions; -use crate::frame::Scope; use crate::function::{IntoPyNativeFunc, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; @@ -56,6 +55,7 @@ use crate::obj::objtype::{self, PyClass, PyClassRef}; use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; +use crate::scope::Scope; use crate::vm::VirtualMachine; use indexmap::IndexMap; diff --git a/vm/src/scope.rs b/vm/src/scope.rs new file mode 100644 index 000000000..7bc34dd0c --- /dev/null +++ b/vm/src/scope.rs @@ -0,0 +1,181 @@ +use std::fmt; +use std::rc::Rc; + +use crate::obj::objdict::PyDictRef; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +/* + * So a scope is a linked list of scopes. + * When a name is looked up, it is check in its scope. + */ +#[derive(Debug)] +struct RcListNode { + elem: T, + next: Option>>, +} + +#[derive(Debug, Clone)] +struct RcList { + head: Option>>, +} + +struct Iter<'a, T: 'a> { + next: Option<&'a RcListNode>, +} + +impl RcList { + pub fn new() -> Self { + RcList { head: None } + } + + pub fn insert(self, elem: T) -> Self { + RcList { + head: Some(Rc::new(RcListNode { + elem, + next: self.head, + })), + } + } + + #[cfg_attr(feature = "flame-it", flame("RcList"))] + pub fn iter(&self) -> Iter { + Iter { + next: self.head.as_ref().map(|node| &**node), + } + } +} + +impl<'a, T> Iterator for Iter<'a, T> { + type Item = &'a T; + + #[cfg_attr(feature = "flame-it", flame("Iter"))] + fn next(&mut self) -> Option { + self.next.map(|node| { + self.next = node.next.as_ref().map(|node| &**node); + &node.elem + }) + } +} + +#[derive(Clone)] +pub struct Scope { + locals: RcList, + pub globals: PyDictRef, +} + +impl fmt::Debug for Scope { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow + f.write_str("Scope") + } +} + +impl Scope { + pub fn new(locals: Option, globals: PyDictRef) -> Scope { + let locals = match locals { + Some(dict) => RcList::new().insert(dict), + None => RcList::new(), + }; + Scope { locals, globals } + } + + pub fn with_builtins( + locals: Option, + globals: PyDictRef, + vm: &VirtualMachine, + ) -> Scope { + if !globals.contains_key("__builtins__", vm) { + globals + .clone() + .set_item("__builtins__", vm.builtins.clone(), vm) + .unwrap(); + } + Scope::new(locals, globals) + } + + pub fn get_locals(&self) -> PyDictRef { + match self.locals.iter().next() { + Some(dict) => dict.clone(), + None => self.globals.clone(), + } + } + + pub fn get_only_locals(&self) -> Option { + self.locals.iter().next().cloned() + } + + pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope { + Scope { + locals: self.locals.clone().insert(locals), + globals: self.globals.clone(), + } + } + + pub fn new_child_scope(&self, ctx: &PyContext) -> Scope { + self.new_child_scope_with_locals(ctx.new_dict()) + } +} + +pub trait NameProtocol { + fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); + fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult; + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); +} + +impl NameProtocol for Scope { + #[cfg_attr(feature = "flame-it", flame("Scope"))] + fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { + for dict in self.locals.iter() { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { + return Some(value); + } + } + + if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { + return Some(value); + } + + vm.get_attribute(vm.builtins.clone(), name).ok() + } + + #[cfg_attr(feature = "flame-it", flame("Scope"))] + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { + for dict in self.locals.iter().skip(1) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { + return Some(value); + } + } + None + } + + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.locals + .iter() + .nth(1) + .expect("no outer scope for non-local") + .set_item(name, value, vm) + .unwrap(); + } + + fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { + self.get_locals().set_item(key, value, vm).unwrap(); + } + + fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult { + self.get_locals().del_item(key, vm) + } + + #[cfg_attr(feature = "flame-it", flame("Scope"))] + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { + self.globals.get_item_option(name, vm).unwrap() + } + + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.globals.set_item(name, value, vm).unwrap(); + } +} diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 01ea2d2ac..c36b1287e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -13,7 +13,7 @@ use std::sync::{Mutex, MutexGuard}; use crate::builtins; use crate::bytecode; -use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; +use crate::frame::{ExecutionResult, Frame, FrameRef}; use crate::frozen; use crate::function::PyFuncArgs; use crate::import; @@ -35,6 +35,7 @@ use crate::pyobject::{ IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; +use crate::scope::Scope; use crate::stdlib; use crate::sysmodule; use num_bigint::BigInt; diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 75b110730..525d8addc 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -6,7 +6,7 @@ use js_sys::{Object, Reflect, SyntaxError, TypeError}; use wasm_bindgen::prelude::*; use rustpython_compiler::compile; -use rustpython_vm::frame::{NameProtocol, Scope}; +use rustpython_vm::scope::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::import; use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue};