Split off bytecode compilation into a separate crate

Cargo.lock

@@ -898,6 +898,18 @@ dependencies = [
  "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rustpython_compiler"
+version = "0.1.0"
+dependencies = [
+ "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-complex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustpython_parser 0.0.1",
+ "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustpython_derive"
 version = "0.1.0"
@@ -927,7 +939,6 @@ name = "rustpython_vm"
 version = "0.1.0"
 dependencies = [
  "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -946,6 +957,7 @@ dependencies = [
  "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustpython_compiler 0.1.0",
  "rustpython_derive 0.1.0",
  "rustpython_parser 0.0.1",
  "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Windel Bouwman", "Shing Lyu <shing.lyu@gmail.com>"]
 edition = "2018"
 
 [workspace]
-members = [".", "derive", "vm", "wasm/lib", "parser"]
+members = [".", "derive", "vm", "wasm/lib", "parser", "compiler"]
 
 [[bench]]
 name = "bench"

compiler/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "rustpython_compiler"
+version = "0.1.0"
+authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>"]
+edition = "2018"
+
+[dependencies]
+bitflags = "1.1"
+rustpython_parser = { path = "../parser" }
+serde = { version = "1.0", features = ["derive"] }
+num-complex = { version = "0.2", features = ["serde"] }
+num-bigint = { version = "0.2", features = ["serde"] }
+log = "0.3"

compiler/src/bytecode.rs

@@ -5,6 +5,7 @@
  * Primitive instruction type, which can be encoded and decoded.
  */
 
+use bitflags::bitflags;
 use num_bigint::BigInt;
 use num_complex::Complex64;
 use rustpython_parser::ast;

compiler/src/compile.rs

@@ -7,11 +7,7 @@
 
 use crate::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use crate::error::{CompileError, CompileErrorType};
-use crate::obj::objcode;
-use crate::obj::objcode::PyCodeRef;
-use crate::pyobject::PyValue;
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope};
-use crate::VirtualMachine;
 use num_complex::Complex64;
 use rustpython_parser::{ast, parser};
 
@@ -27,12 +23,7 @@ struct Compiler {
 }
 
 /// Compile a given sourcecode into a bytecode object.
-pub fn compile(
-    vm: &VirtualMachine,
-    source: &str,
-    mode: &Mode,
-    source_path: String,
-) -> Result<PyCodeRef, CompileError> {
+pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
     let mut compiler = Compiler::new();
     compiler.source_path = Some(source_path);
     compiler.push_new_code_object("<module>".to_string());
@@ -57,7 +48,7 @@ pub fn compile(
 
     let code = compiler.pop_code_object();
     trace!("Compilation completed: {:?}", code);
-    Ok(objcode::PyCode::new(code).into_ref(vm))
+    Ok(code)
 }
 
 pub enum Mode {

compiler/src/lib.rs

@@ -0,0 +1,7 @@
+#[macro_use]
+extern crate log;
+
+pub mod bytecode;
+pub mod compile;
+pub mod error;
+mod symboltable;

src/main.rs

@@ -75,7 +75,8 @@ fn main() {
 }
 
 fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult {
-    let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path.clone())
+    let code_obj = vm
+        .compile(source, &compile::Mode::Exec, source_path.clone())
         .map_err(|err| vm.new_syntax_error(&err))?;
     // trace!("Code object: {:?}", code_obj.borrow());
     let attrs = vm.ctx.new_dict();
@@ -160,7 +161,7 @@ fn test_run_script() {
 }
 
 fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> {
-    match compile::compile(vm, source, &compile::Mode::Single, "<stdin>".to_string()) {
+    match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
         Ok(code) => {
             match vm.run_code_obj(code, scope.clone()) {
                 Ok(value) => {

vm/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Shing Lyu <shing.lyu@gmail.com>"]
 edition = "2018"
 
 [dependencies]
-bitflags = "1.0.4"
 num-complex = { version = "0.2", features = ["serde"] }
 num-bigint = { version = "0.2.1", features = ["serde"] }
 num-traits = "0.2"
@@ -15,6 +14,7 @@ rand = "0.5"
 log = "0.3"
 rustpython_derive = {path = "../derive"}
 rustpython_parser = {path = "../parser"}
+rustpython_compiler = {path = "../compiler"}
 serde = { version = "1.0.66", features = ["derive"] }
 serde_json = "1.0.26"
 byteorder = "1.2.6"

vm/src/builtins.rs

@@ -121,7 +121,7 @@ fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult<PyCodeRef
         }
     };
 
-    compile::compile(vm, &source, &mode, args.filename.value.to_string())
+    vm.compile(&source, &mode, args.filename.value.to_string())
         .map_err(|err| vm.new_syntax_error(&err))
 }
 
@@ -170,7 +170,7 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
         let source = objstr::get_value(source);
         // TODO: fix this newline bug:
         let source = format!("{}\n", source);
-        compile::compile(vm, &source, &mode, "<string>".to_string())
+        vm.compile(&source, &mode, "<string>".to_string())
             .map_err(|err| vm.new_syntax_error(&err))?
     } else {
         return Err(vm.new_type_error("code argument must be str or code object".to_string()));
@@ -198,7 +198,7 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
         let source = objstr::get_value(source);
         // TODO: fix this newline bug:
         let source = format!("{}\n", source);
-        compile::compile(vm, &source, &mode, "<string>".to_string())
+        vm.compile(&source, &mode, "<string>".to_string())
             .map_err(|err| vm.new_syntax_error(&err))?
     } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) {
         code_obj

vm/src/eval.rs

@@ -6,7 +6,7 @@ use crate::pyobject::PyResult;
 use crate::vm::VirtualMachine;
 
 pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult {
-    match compile::compile(vm, source, &compile::Mode::Eval, source_path.to_string()) {
+    match vm.compile(source, &compile::Mode::Eval, source_path.to_string()) {
         Ok(bytecode) => {
             debug!("Code object: {:?}", bytecode);
             vm.run_code_obj(bytecode, scope)

vm/src/import.rs

@@ -84,7 +84,8 @@ pub fn import_file(
     content: String,
 ) -> PyResult {
     let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap();
-    let code_obj = compile::compile(vm, &content, &compile::Mode::Exec, file_path.clone())
+    let code_obj = vm
+        .compile(&content, &compile::Mode::Exec, file_path.clone())
         .map_err(|err| vm.new_syntax_error(&err))?;
     // trace!("Code object: {:?}", code_obj);
 

vm/src/lib.rs

@@ -12,8 +12,6 @@
     clippy::implicit_hasher
 )]
 
-#[macro_use]
-extern crate bitflags;
 #[macro_use]
 extern crate lazy_static;
 extern crate lexical;
@@ -37,10 +35,7 @@ pub use rustpython_derive::*;
 pub mod macros;
 
 mod builtins;
-pub mod bytecode;
-pub mod compile;
 mod dictdatatype;
-pub mod error;
 pub mod eval;
 mod exceptions;
 pub mod format;
@@ -52,7 +47,6 @@ pub mod obj;
 mod pyhash;
 pub mod pyobject;
 pub mod stdlib;
-mod symboltable;
 mod sysmodule;
 mod traceback;
 pub mod util;
@@ -61,3 +55,4 @@ mod vm;
 // pub use self::pyobject::Executor;
 pub use self::exceptions::print_exception;
 pub use self::vm::VirtualMachine;
+pub use rustpython_compiler::*;

vm/src/stdlib/imp.rs

@@ -58,13 +58,8 @@ fn imp_exec_builtin(_mod: PyModuleRef, _vm: &VirtualMachine) -> i32 {
 fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult<PyCodeRef> {
     let name_str = name.as_str();
     if let Some(frozen) = vm.frozen.borrow().get(name_str) {
-        compile::compile(
-            vm,
-            frozen,
-            &compile::Mode::Exec,
-            format!("frozen {}", name_str),
-        )
-        .map_err(|err| vm.new_syntax_error(&err))
+        vm.compile(frozen, &compile::Mode::Exec, format!("frozen {}", name_str))
+            .map_err(|err| vm.new_syntax_error(&err))
     } else {
         Err(vm.new_import_error(format!("No such frozen object named {}", name.as_str())))
     }

vm/src/vm.rs

@@ -4,8 +4,6 @@
 //!   https://github.com/ProgVal/pythonvm-rust/blob/master/src/processor/mod.rs
 //!
 
-extern crate rustpython_parser;
-
 use std::cell::{Ref, RefCell};
 use std::collections::hash_map::HashMap;
 use std::collections::hash_set::HashSet;
@@ -14,13 +12,14 @@ use std::sync::{Mutex, MutexGuard};
 
 use crate::builtins;
 use crate::bytecode;
+use crate::compile;
 use crate::error::CompileError;
 use crate::frame::{ExecutionResult, Frame, FrameRef, Scope};
 use crate::frozen;
 use crate::function::PyFuncArgs;
 use crate::obj::objbool;
 use crate::obj::objbuiltinfunc::PyBuiltinFunction;
-use crate::obj::objcode::PyCodeRef;
+use crate::obj::objcode::{PyCode, PyCodeRef};
 use crate::obj::objdict::PyDictRef;
 use crate::obj::objfunction::{PyFunction, PyMethod};
 use crate::obj::objgenerator::PyGenerator;
@@ -736,6 +735,16 @@ impl VirtualMachine {
         )
     }
 
+    pub fn compile(
+        &self,
+        source: &str,
+        mode: &compile::Mode,
+        source_path: String,
+    ) -> Result<PyCodeRef, CompileError> {
+        compile::compile(source, mode, source_path)
+            .map(|codeobj| PyCode::new(codeobj).into_ref(self))
+    }
+
     pub fn _sub(&self, a: PyObjectRef, b: PyObjectRef) -> PyResult {
         self.call_or_reflection(a, b, "__sub__", "__rsub__", |vm, a, b| {
             Err(vm.new_unsupported_operand_error(a, b, "-"))

wasm/lib/src/vm_class.rs

@@ -263,7 +263,7 @@ impl WASMVirtualMachine {
                  ref vm, ref scope, ..
              }| {
                 source.push('\n');
-                let code = compile::compile(vm, &source, &mode, "<wasm>".to_string());
+                let code = vm.compile(&source, &mode, "<wasm>".to_string());
                 let code = code.map_err(|err| {
                     let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));
                     if let rustpython_vm::error::CompileErrorType::Parse(ref parse_error) =