Move Scope struct to its own file, making the frame module private in the process.

DEVELOPMENT.md

@@ -2,11 +2,16 @@
 
 ## Code organization
 
+- `bytecode/src`: python bytecode representation in rust structures
+- `compiler/src`: python compilation to bytecode
 - `parser/src`: python lexing, parsing and ast
+- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
+   the python side of the standard library.
 - `vm/src`: python virtual machine
   - `builtins.rs`: Builtin functions
   - `compile.rs`: the python compiler from ast to bytecode
   - `obj`: python builtin types
+  - `stdlib`: Standard library parts implemented in rust.
 - `src`: using the other subcrates to bring rustpython to life.
 - `docs`: documentation (work in progress)
 - `py_code_object`: CPython bytecode to rustpython bytecode converter (work in
@@ -28,8 +33,7 @@ To test rustpython, there is a collection of python snippets located in the
 
 ```shell
 $ cd tests
-$ pipenv install
-$ pipenv run pytest -v
+$ pytest -v
 ```
 
 There also are some unit tests, you can run those with cargo:

README.md

@@ -81,7 +81,8 @@ Documentation HTML files can then be found in the `target/doc` directory.
 ## Contributing
 
 Contributions are more than welcome, and in many cases we are happy to guide
-contributors through PRs or on gitter.
+contributors through PRs or on gitter. Please refer to the
+[development guide](DEVELOPMENT.md) as well for tips on developments.
 
 With that in mind, please note this project is maintained by volunteers, some of
 the best ways to get started are below:

src/main.rs

@@ -8,11 +8,11 @@ use clap::{App, Arg, ArgMatches};
 use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
 use rustpython_parser::error::ParseErrorType;
 use rustpython_vm::{
-    frame::Scope,
     import,
     obj::objstr,
     print_exception,
     pyobject::{ItemProtocol, PyResult},
+    scope::Scope,
     util, PySettings, VirtualMachine,
 };
 use std::convert::TryInto;

vm/src/builtins.rs

@@ -22,12 +22,12 @@ use crate::obj::objtype::{self, PyClassRef};
 use rustpython_compiler::compile;
 
 use crate::eval::get_compile_mode;
-use crate::frame::Scope;
 use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs};
 use crate::pyobject::{
     Either, IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue,
     TryFromObject, TypeProtocol,
 };
+use crate::scope::Scope;
 use crate::vm::VirtualMachine;
 
 use crate::obj::objbyteinner::PyByteInner;

vm/src/eval.rs

@@ -1,5 +1,5 @@
-use crate::frame::Scope;
 use crate::pyobject::PyResult;
+use crate::scope::Scope;
 use crate::vm::VirtualMachine;
 use rustpython_compiler::compile;
 

vm/src/frame.rs

@@ -1,6 +1,5 @@
 use std::cell::RefCell;
 use std::fmt;
-use std::rc::Rc;
 
 use crate::builtins;
 use crate::bytecode;
@@ -17,188 +16,13 @@ use crate::obj::objtuple::PyTuple;
 use crate::obj::objtype;
 use crate::obj::objtype::PyClassRef;
 use crate::pyobject::{
-    IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
-    TypeProtocol,
+    IdProtocol, ItemProtocol, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol,
 };
+use crate::scope::{NameProtocol, Scope};
 use crate::vm::VirtualMachine;
 use indexmap::IndexMap;
 use itertools::Itertools;
 
-/*
- * So a scope is a linked list of scopes.
- * When a name is looked up, it is check in its scope.
- */
-#[derive(Debug)]
-struct RcListNode<T> {
-    elem: T,
-    next: Option<Rc<RcListNode<T>>>,
-}
-
-#[derive(Debug, Clone)]
-struct RcList<T> {
-    head: Option<Rc<RcListNode<T>>>,
-}
-
-struct Iter<'a, T: 'a> {
-    next: Option<&'a RcListNode<T>>,
-}
-
-impl<T> RcList<T> {
-    pub fn new() -> Self {
-        RcList { head: None }
-    }
-
-    pub fn insert(self, elem: T) -> Self {
-        RcList {
-            head: Some(Rc::new(RcListNode {
-                elem,
-                next: self.head,
-            })),
-        }
-    }
-
-    #[cfg_attr(feature = "flame-it", flame("RcList"))]
-    pub fn iter(&self) -> Iter<T> {
-        Iter {
-            next: self.head.as_ref().map(|node| &**node),
-        }
-    }
-}
-
-impl<'a, T> Iterator for Iter<'a, T> {
-    type Item = &'a T;
-
-    #[cfg_attr(feature = "flame-it", flame("Iter"))]
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next.map(|node| {
-            self.next = node.next.as_ref().map(|node| &**node);
-            &node.elem
-        })
-    }
-}
-
-#[derive(Clone)]
-pub struct Scope {
-    locals: RcList<PyDictRef>,
-    pub globals: PyDictRef,
-}
-
-impl fmt::Debug for Scope {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow
-        f.write_str("Scope")
-    }
-}
-
-impl Scope {
-    pub fn new(locals: Option<PyDictRef>, globals: PyDictRef) -> Scope {
-        let locals = match locals {
-            Some(dict) => RcList::new().insert(dict),
-            None => RcList::new(),
-        };
-        Scope { locals, globals }
-    }
-
-    pub fn with_builtins(
-        locals: Option<PyDictRef>,
-        globals: PyDictRef,
-        vm: &VirtualMachine,
-    ) -> Scope {
-        if !globals.contains_key("__builtins__", vm) {
-            globals
-                .clone()
-                .set_item("__builtins__", vm.builtins.clone(), vm)
-                .unwrap();
-        }
-        Scope::new(locals, globals)
-    }
-
-    pub fn get_locals(&self) -> PyDictRef {
-        match self.locals.iter().next() {
-            Some(dict) => dict.clone(),
-            None => self.globals.clone(),
-        }
-    }
-
-    pub fn get_only_locals(&self) -> Option<PyDictRef> {
-        self.locals.iter().next().cloned()
-    }
-
-    pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope {
-        Scope {
-            locals: self.locals.clone().insert(locals),
-            globals: self.globals.clone(),
-        }
-    }
-
-    pub fn new_child_scope(&self, ctx: &PyContext) -> Scope {
-        self.new_child_scope_with_locals(ctx.new_dict())
-    }
-}
-
-pub trait NameProtocol {
-    fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
-    fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
-    fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult;
-    fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
-    fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
-    fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
-    fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
-}
-
-impl NameProtocol for Scope {
-    #[cfg_attr(feature = "flame-it", flame("Scope"))]
-    fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
-        for dict in self.locals.iter() {
-            if let Some(value) = dict.get_item_option(name, vm).unwrap() {
-                return Some(value);
-            }
-        }
-
-        if let Some(value) = self.globals.get_item_option(name, vm).unwrap() {
-            return Some(value);
-        }
-
-        vm.get_attribute(vm.builtins.clone(), name).ok()
-    }
-
-    #[cfg_attr(feature = "flame-it", flame("Scope"))]
-    fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
-        for dict in self.locals.iter().skip(1) {
-            if let Some(value) = dict.get_item_option(name, vm).unwrap() {
-                return Some(value);
-            }
-        }
-        None
-    }
-
-    fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) {
-        self.locals
-            .iter()
-            .nth(1)
-            .expect("no outer scope for non-local")
-            .set_item(name, value, vm)
-            .unwrap();
-    }
-
-    fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) {
-        self.get_locals().set_item(key, value, vm).unwrap();
-    }
-
-    fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult {
-        self.get_locals().del_item(key, vm)
-    }
-
-    #[cfg_attr(feature = "flame-it", flame("Scope"))]
-    fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
-        self.globals.get_item_option(name, vm).unwrap()
-    }
-
-    fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) {
-        self.globals.set_item(name, value, vm).unwrap();
-    }
-}
-
 #[derive(Clone, Debug)]
 struct Block {
     /// The type of block.

vm/src/import.rs

@@ -3,9 +3,9 @@
  */
 
 use crate::bytecode::CodeObject;
-use crate::frame::Scope;
 use crate::obj::{objcode, objsequence, objstr, objtype};
 use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue};
+use crate::scope::Scope;
 use crate::vm::VirtualMachine;
 #[cfg(feature = "rustpython-compiler")]
 use rustpython_compiler::compile;

vm/src/lib.rs

@@ -56,7 +56,7 @@ mod dictdatatype;
 pub mod eval;
 mod exceptions;
 pub mod format;
-pub mod frame;
+mod frame;
 mod frozen;
 pub mod function;
 pub mod import;
@@ -64,6 +64,7 @@ pub mod obj;
 pub mod py_serde;
 mod pyhash;
 pub mod pyobject;
+pub mod scope;
 pub mod stdlib;
 mod sysmodule;
 mod traceback;

vm/src/obj/objfunction.rs

@@ -1,10 +1,10 @@
-use crate::frame::Scope;
 use crate::function::{Args, KwArgs};
 use crate::obj::objcode::PyCodeRef;
 use crate::obj::objdict::PyDictRef;
 use crate::obj::objtuple::PyTupleRef;
 use crate::obj::objtype::PyClassRef;
 use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol};
+use crate::scope::Scope;
 use crate::vm::VirtualMachine;
 
 pub type PyFunctionRef = PyRef<PyFunction>;

vm/src/obj/objsuper.rs

@@ -6,7 +6,6 @@ https://github.com/python/cpython/blob/50b48572d9a90c5bb36e2bef6179548ea927a35a/
 
 */
 
-use crate::frame::NameProtocol;
 use crate::function::{OptionalArg, PyFuncArgs};
 use crate::obj::objfunction::PyMethod;
 use crate::obj::objstr;
@@ -14,6 +13,7 @@ use crate::obj::objtype::{PyClass, PyClassRef};
 use crate::pyobject::{
     ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol,
 };
+use crate::scope::NameProtocol;
 use crate::vm::VirtualMachine;
 
 use super::objtype;

vm/src/pyobject.rs

@@ -15,7 +15,6 @@ use num_traits::{One, Zero};
 
 use crate::bytecode;
 use crate::exceptions;
-use crate::frame::Scope;
 use crate::function::{IntoPyNativeFunc, PyFuncArgs};
 use crate::obj::objbool;
 use crate::obj::objbuiltinfunc::PyBuiltinFunction;
@@ -56,6 +55,7 @@ use crate::obj::objtype::{self, PyClass, PyClassRef};
 use crate::obj::objweakproxy;
 use crate::obj::objweakref;
 use crate::obj::objzip;
+use crate::scope::Scope;
 use crate::vm::VirtualMachine;
 use indexmap::IndexMap;
 

vm/src/scope.rs

@@ -0,0 +1,181 @@
+use std::fmt;
+use std::rc::Rc;
+
+use crate::obj::objdict::PyDictRef;
+use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult};
+use crate::vm::VirtualMachine;
+
+/*
+ * So a scope is a linked list of scopes.
+ * When a name is looked up, it is check in its scope.
+ */
+#[derive(Debug)]
+struct RcListNode<T> {
+    elem: T,
+    next: Option<Rc<RcListNode<T>>>,
+}
+
+#[derive(Debug, Clone)]
+struct RcList<T> {
+    head: Option<Rc<RcListNode<T>>>,
+}
+
+struct Iter<'a, T: 'a> {
+    next: Option<&'a RcListNode<T>>,
+}
+
+impl<T> RcList<T> {
+    pub fn new() -> Self {
+        RcList { head: None }
+    }
+
+    pub fn insert(self, elem: T) -> Self {
+        RcList {
+            head: Some(Rc::new(RcListNode {
+                elem,
+                next: self.head,
+            })),
+        }
+    }
+
+    #[cfg_attr(feature = "flame-it", flame("RcList"))]
+    pub fn iter(&self) -> Iter<T> {
+        Iter {
+            next: self.head.as_ref().map(|node| &**node),
+        }
+    }
+}
+
+impl<'a, T> Iterator for Iter<'a, T> {
+    type Item = &'a T;
+
+    #[cfg_attr(feature = "flame-it", flame("Iter"))]
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next.map(|node| {
+            self.next = node.next.as_ref().map(|node| &**node);
+            &node.elem
+        })
+    }
+}
+
+#[derive(Clone)]
+pub struct Scope {
+    locals: RcList<PyDictRef>,
+    pub globals: PyDictRef,
+}
+
+impl fmt::Debug for Scope {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow
+        f.write_str("Scope")
+    }
+}
+
+impl Scope {
+    pub fn new(locals: Option<PyDictRef>, globals: PyDictRef) -> Scope {
+        let locals = match locals {
+            Some(dict) => RcList::new().insert(dict),
+            None => RcList::new(),
+        };
+        Scope { locals, globals }
+    }
+
+    pub fn with_builtins(
+        locals: Option<PyDictRef>,
+        globals: PyDictRef,
+        vm: &VirtualMachine,
+    ) -> Scope {
+        if !globals.contains_key("__builtins__", vm) {
+            globals
+                .clone()
+                .set_item("__builtins__", vm.builtins.clone(), vm)
+                .unwrap();
+        }
+        Scope::new(locals, globals)
+    }
+
+    pub fn get_locals(&self) -> PyDictRef {
+        match self.locals.iter().next() {
+            Some(dict) => dict.clone(),
+            None => self.globals.clone(),
+        }
+    }
+
+    pub fn get_only_locals(&self) -> Option<PyDictRef> {
+        self.locals.iter().next().cloned()
+    }
+
+    pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope {
+        Scope {
+            locals: self.locals.clone().insert(locals),
+            globals: self.globals.clone(),
+        }
+    }
+
+    pub fn new_child_scope(&self, ctx: &PyContext) -> Scope {
+        self.new_child_scope_with_locals(ctx.new_dict())
+    }
+}
+
+pub trait NameProtocol {
+    fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
+    fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
+    fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult;
+    fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
+    fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
+    fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef>;
+    fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef);
+}
+
+impl NameProtocol for Scope {
+    #[cfg_attr(feature = "flame-it", flame("Scope"))]
+    fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
+        for dict in self.locals.iter() {
+            if let Some(value) = dict.get_item_option(name, vm).unwrap() {
+                return Some(value);
+            }
+        }
+
+        if let Some(value) = self.globals.get_item_option(name, vm).unwrap() {
+            return Some(value);
+        }
+
+        vm.get_attribute(vm.builtins.clone(), name).ok()
+    }
+
+    #[cfg_attr(feature = "flame-it", flame("Scope"))]
+    fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
+        for dict in self.locals.iter().skip(1) {
+            if let Some(value) = dict.get_item_option(name, vm).unwrap() {
+                return Some(value);
+            }
+        }
+        None
+    }
+
+    fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) {
+        self.locals
+            .iter()
+            .nth(1)
+            .expect("no outer scope for non-local")
+            .set_item(name, value, vm)
+            .unwrap();
+    }
+
+    fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) {
+        self.get_locals().set_item(key, value, vm).unwrap();
+    }
+
+    fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult {
+        self.get_locals().del_item(key, vm)
+    }
+
+    #[cfg_attr(feature = "flame-it", flame("Scope"))]
+    fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
+        self.globals.get_item_option(name, vm).unwrap()
+    }
+
+    fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) {
+        self.globals.set_item(name, value, vm).unwrap();
+    }
+}

vm/src/vm.rs

@@ -13,7 +13,7 @@ use std::sync::{Mutex, MutexGuard};
 
 use crate::builtins;
 use crate::bytecode;
-use crate::frame::{ExecutionResult, Frame, FrameRef, Scope};
+use crate::frame::{ExecutionResult, Frame, FrameRef};
 use crate::frozen;
 use crate::function::PyFuncArgs;
 use crate::import;
@@ -35,6 +35,7 @@ use crate::pyobject::{
     IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef,
     TypeProtocol,
 };
+use crate::scope::Scope;
 use crate::stdlib;
 use crate::sysmodule;
 use num_bigint::BigInt;

wasm/lib/src/vm_class.rs

@@ -6,7 +6,7 @@ use js_sys::{Object, Reflect, SyntaxError, TypeError};
 use wasm_bindgen::prelude::*;
 
 use rustpython_compiler::compile;
-use rustpython_vm::frame::{NameProtocol, Scope};
+use rustpython_vm::scope::{NameProtocol, Scope};
 use rustpython_vm::function::PyFuncArgs;
 use rustpython_vm::import;
 use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue};