Add support for marshal with loads and dumps

Cargo.lock

@@ -78,6 +78,16 @@ dependencies = [
  "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "bincode"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.1"
@@ -509,6 +519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -517,6 +528,7 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -875,6 +887,7 @@ dependencies = [
  "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -883,6 +896,7 @@ dependencies = [
 name = "rustpython_vm"
 version = "0.1.0"
 dependencies = [
+ "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -904,7 +918,6 @@ dependencies = [
  "rustpython_derive 0.1.0",
  "rustpython_parser 0.0.1",
  "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
  "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)",
@@ -970,6 +983,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "serde"
 version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "serde_derive"
@@ -1423,6 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799"
 "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4"
 "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6"
+"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7"
 "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
 "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"

parser/Cargo.toml

@@ -16,3 +16,4 @@ num-bigint = "0.2"
 num-traits = "0.2"
 unicode-xid = "0.1.0"
 unic-emoji-char = "0.9.0"
+serde = { version = "1.0.66", features = ["derive"] }

parser/src/ast.rs

@@ -4,6 +4,8 @@
 
 pub use super::lexer::Location;
 use num_bigint::BigInt;
+use serde::{Serialize, Deserialize};
+
 /*
 #[derive(Debug)]
 
@@ -381,7 +383,7 @@ pub enum Number {
 }
 
 /// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq)]
+#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum ConversionFlag {
     /// Converts by calling `str(<value>)`.
     Str,

parser/src/lexer.rs

@@ -12,6 +12,7 @@ use std::collections::HashMap;
 use std::str::FromStr;
 use unic_emoji_char::is_emoji_presentation;
 use unicode_xid::UnicodeXID;
+use serde::{Serialize, Deserialize};
 
 #[derive(Clone, Copy, PartialEq, Debug)]
 struct IndentationLevel {
@@ -71,7 +72,7 @@ pub enum LexicalErrorType {
     OtherError(String),
 }
 
-#[derive(Clone, Debug, Default, PartialEq)]
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct Location {
     row: usize,
     column: usize,

tests/snippets/stdlib_marshal.py

@@ -0,0 +1,7 @@
+import marshal
+orig = compile("1 + 1", "", 'eval')
+
+dumped = marshal.dumps(orig)
+loaded = marshal.loads(dumped)
+
+assert eval(loaded) == eval(orig)

vm/Cargo.toml

@@ -6,8 +6,8 @@ edition = "2018"
 
 [dependencies]
 bitflags = "1.0.4"
-num-complex = "0.2"
-num-bigint = "0.2.1"
+num-complex = { version = "0.2", features = ["serde"] }
+num-bigint = { version = "0.2.1", features = ["serde"] }
 num-traits = "0.2"
 num-integer = "0.1.39"
 num-rational = "0.2.1"
@@ -15,8 +15,7 @@ rand = "0.5"
 log = "0.3"
 rustpython_derive = {path = "../derive"}
 rustpython_parser = {path = "../parser"}
-serde = "1.0.66"
-serde_derive = "1.0.66"
+serde = { version = "1.0.66", features = ["derive"] }
 serde_json = "1.0.26"
 byteorder = "1.2.6"
 regex = "1"
@@ -30,6 +29,7 @@ lexical = "2.0.0"
 itertools = "^0.8.0"
 hexf = "0.1.0"
 indexmap = "1.0.2"
+bincode = "1.1.4"
 
 # TODO: release and publish to crates.io
 [dependencies.unicode-casing]

vm/src/bytecode.rs

@@ -10,10 +10,11 @@ use num_complex::Complex64;
 use rustpython_parser::ast;
 use std::collections::{HashMap, HashSet};
 use std::fmt;
+use serde::{Serialize, Deserialize};
 
 /// Primary container of a single code object. Each python function has
 /// a codeobject. Also a module has a codeobject.
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
 pub struct CodeObject {
     pub instructions: Vec<Instruction>,
     pub label_map: HashMap<Label, usize>,
@@ -29,6 +30,7 @@ pub struct CodeObject {
 }
 
 bitflags! {
+    #[derive(Serialize, Deserialize)]
     pub struct FunctionOpArg: u8 {
         const HAS_DEFAULTS = 0x01;
         const HAS_KW_ONLY_DEFAULTS = 0x02;
@@ -38,7 +40,7 @@ bitflags! {
 
 pub type Label = usize;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum NameScope {
     Local,
     NonLocal,
@@ -46,7 +48,7 @@ pub enum NameScope {
 }
 
 /// A Single bytecode instruction.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Instruction {
     Import {
         name: String,
@@ -187,14 +189,14 @@ pub enum Instruction {
 
 use self::Instruction::*;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum CallType {
     Positional(usize),
     Keyword(usize),
     Ex(bool),
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Constant {
     Integer { value: BigInt },
     Float { value: f64 },
@@ -208,7 +210,7 @@ pub enum Constant {
     Ellipsis,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum ComparisonOperator {
     Greater,
     GreaterOrEqual,
@@ -222,7 +224,7 @@ pub enum ComparisonOperator {
     IsNot,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum BinaryOperator {
     Power,
     Multiply,
@@ -240,7 +242,7 @@ pub enum BinaryOperator {
     Or,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum UnaryOperator {
     Not,
     Invert,
@@ -248,7 +250,7 @@ pub enum UnaryOperator {
     Plus,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Varargs {
     None,
     Unnamed,

vm/src/pyobject.rs

@@ -294,7 +294,7 @@ impl PyContext {
         let zip_type = create_type("zip", &type_type, &object_type);
         let bool_type = create_type("bool", &type_type, &int_type);
         let memoryview_type = create_type("memoryview", &type_type, &object_type);
-        let code_type = create_type("code", &type_type, &int_type);
+        let code_type = create_type("code", &type_type, &object_type);
         let range_type = create_type("range", &type_type, &object_type);
         let rangeiterator_type = create_type("range_iterator", &type_type, &object_type);
         let slice_type = create_type("slice", &type_type, &object_type);

vm/src/stdlib/marshal.rs

@@ -0,0 +1,24 @@
+use crate::obj::objcode::{PyCodeRef, PyCode};
+use crate::pyobject::{PyObjectRef, PyResult, IntoPyObject};
+use crate::vm::VirtualMachine;
+use crate::obj::objbytes::{PyBytes, PyBytesRef};
+use crate::bytecode;
+
+fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult {
+    PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm)
+}
+
+fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult {
+    let code = bincode::deserialize::<bytecode::CodeObject>(&code_bytes).unwrap();
+    let pycode = PyCode { code };
+    pycode.into_pyobject(vm)
+}
+
+pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+    let ctx = &vm.ctx;
+
+    py_module!(vm, "marshal", {
+        "loads" => ctx.new_rustfunc(marshal_loads),
+        "dumps" => ctx.new_rustfunc(marshal_dumps)
+    })
+}

vm/src/stdlib/mod.rs

@@ -5,6 +5,7 @@ mod imp;
 mod itertools;
 pub(crate) mod json;
 mod keyword;
+mod marshal;
 mod math;
 mod platform;
 mod pystruct;
@@ -42,6 +43,7 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
     modules.insert("itertools".to_string(), Box::new(itertools::make_module));
     modules.insert("json".to_string(), Box::new(json::make_module));
     modules.insert("keyword".to_string(), Box::new(keyword::make_module));
+    modules.insert("marshal".to_string(), Box::new(marshal::make_module));
     modules.insert("math".to_string(), Box::new(math::make_module));
     modules.insert("platform".to_string(), Box::new(platform::make_module));
     modules.insert("re".to_string(), Box::new(re::make_module));