From 310f89ec215979bab0729638c6bad7c018bb0704 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 7 Jul 2019 15:23:43 +0200 Subject: [PATCH 1/2] Remove bytecode compilation. Add feature to enable verbose logging in vm. --- tests/compile_code.py | 60 ----------------------------------------- tests/test_snippets.py | 19 ------------- vm/Cargo.toml | 1 + vm/src/frame.rs | 5 +++- vm/src/macros.rs | 11 ++++++++ vm/src/obj/objnone.rs | 2 +- vm/src/obj/objobject.rs | 4 +-- vm/src/obj/objtype.rs | 8 +++--- vm/src/vm.rs | 11 ++++---- 9 files changed, 29 insertions(+), 92 deletions(-) delete mode 100644 tests/compile_code.py diff --git a/tests/compile_code.py b/tests/compile_code.py deleted file mode 100644 index 055b099ab..000000000 --- a/tests/compile_code.py +++ /dev/null @@ -1,60 +0,0 @@ -import bytecode -import sys -import json -import types - - -class CodeEncoder(json.JSONEncoder): - def default(self, obj): - if (isinstance(obj, types.CodeType)): - return serialize_code(obj) - return json.JSONEncoder.default(self, obj) - -def serialize_code(code): - c = bytecode.Bytecode().from_code(code).to_concrete_bytecode() - return ( - { - "co_consts": consts_to_rust_enum(c.consts), - "co_names": c.names, - "co_name": c.name, - "co_code": parse_co_code_to_str(c), - "co_varnames": c.varnames - } - ) - - -def consts_to_rust_enum(consts): - def capitalize_first(s): - return s[0].upper() + s[1:] - - def const_to_rust_enum(const): - if type(const).__name__ == "tuple": - return {capitalize_first(str(type(const).__name__)): list(map(const_to_rust_enum, const))} - else: - return {capitalize_first(str(type(const).__name__)): const} - return list(map(const_to_rust_enum, consts)) - - -def parse_co_code_to_str(c): - return list( - map(lambda op: (op.size, op.name, op.arg if op.arg != bytecode.UNSET else None), - c) - ) - - -def compile_to_bytecode(filename, out_file=None): - with open(filename, 'rU') as f: - code = f.read() - - code = compile(code, filename, "exec") - - print(CodeEncoder(indent=4).encode(code), file=out_file) - - -def main(): - filename = sys.argv[1] - compile_to_bytecode(filename) - - -if __name__ == "__main__": - main() diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 17bcf0dca..6e5b4a915 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -13,8 +13,6 @@ import enum from pathlib import Path import shutil -import compile_code - class _TestType(enum.Enum): functional = 1 @@ -40,8 +38,6 @@ def perform_test(filename, method, test_type): logger.info("Running %s via %s", filename, method) if method == "cpython": run_via_cpython(filename) - elif method == "cpython_bytecode": - run_via_cpython_bytecode(filename, test_type) elif method == "rustpython": run_via_rustpython(filename, test_type) else: @@ -54,20 +50,6 @@ def run_via_cpython(filename): subprocess.check_call([sys.executable, filename], env=env) -def run_via_cpython_bytecode(filename, test_type): - # Step1: Create bytecode file: - bytecode_filename = filename + ".bytecode" - with open(bytecode_filename, "w") as f: - compile_code.compile_to_bytecode(filename, out_file=f) - - # Step2: run cpython bytecode: - env = os.environ.copy() - env["RUST_LOG"] = "info,cargo=error,jobserver=error" - env["RUST_BACKTRACE"] = "1" - with pushd(CPYTHON_RUNNER_DIR): - subprocess.check_call(["cargo", "run", bytecode_filename], env=env) - - def run_via_rustpython(filename, test_type): env = os.environ.copy() env['RUST_LOG'] = 'info,cargo=error,jobserver=error' @@ -142,7 +124,6 @@ def generate_slices(path): @populate("cpython") -# @populate('cpython_bytecode') @populate("rustpython") class SampleTestCase(unittest.TestCase): @classmethod diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 110fb6f45..4f09db2ff 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [features] default = ["rustpython-parser", "rustpython-compiler"] +vm-tracing-logging = [] [dependencies] # Crypto: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 61ea0a78a..5e5cdcea8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -295,7 +295,7 @@ impl Frame { let traceback = vm .get_attribute(exception.clone(), "__traceback__") .unwrap(); - trace!("Adding to traceback: {:?} {:?}", traceback, lineno); + vm_trace!("Adding to traceback: {:?} {:?}", traceback, lineno); let raise_location = vm.ctx.new_tuple(vec![ vm.ctx.new_str(filename.clone()), vm.ctx.new_int(lineno.row()), @@ -335,6 +335,8 @@ impl Frame { /// Execute a single instruction. fn execute_instruction(&self, vm: &VirtualMachine) -> FrameResult { let instruction = self.fetch_instruction(); + + #[cfg(feature = "vm-tracing-logging")] { trace!("======="); /* TODO: @@ -1142,6 +1144,7 @@ impl Frame { fn jump(&self, label: bytecode::Label) { let target_pc = self.code.label_map[&label]; + #[cfg(feature = "vm-tracing-logging")] trace!("jump from {:?} to {:?}", self.lasti, target_pc); *self.lasti.borrow_mut() = target_pc; } diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 3e5d3262f..158898c48 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -256,3 +256,14 @@ macro_rules! match_class { } }; } + + +/// Super detailed logging. Might soon overflow your logbuffers +/// Default, this logging is discarded, except when a the `vm-tracing-logging` +/// build feature is enabled. +macro_rules! vm_trace { + ($($arg:tt)+) => { + #[cfg(feature = "vm-tracing-logging")] + trace!($($arg)+); + } +} diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index aff20f465..fa23ca1b0 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -43,7 +43,7 @@ impl PyNoneRef { } fn get_attribute(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { - trace!("None.__getattribute__({:?}, {:?})", self, name); + vm_trace!("None.__getattribute__({:?}, {:?})", self, name); let cls = self.class(); // Properties use a comparision with None to determine if they are either invoked by am diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 77830a402..8f2408dba 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -66,7 +66,7 @@ fn object_setattr( value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - trace!("object.__setattr__({:?}, {}, {:?})", obj, attr_name, value); + vm_trace!("object.__setattr__({:?}, {}, {:?})", obj, attr_name, value); let cls = obj.class(); if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) { @@ -220,7 +220,7 @@ fn object_dict_setter( fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_str.value; - trace!("object.__getattribute__({:?}, {:?})", obj, name); + vm_trace!("object.__getattribute__({:?}, {:?})", obj, name); let cls = obj.class(); if let Some(attr) = objtype::class_get_attr(&cls, &name) { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 90ec95ae8..a0692ff64 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -114,7 +114,7 @@ impl PyClassRef { fn getattribute(self, name_ref: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_ref.value; - trace!("type.__getattribute__({:?}, {:?})", self, name); + vm_trace!("type.__getattribute__({:?}, {:?})", self, name); let mcl = self.class(); if let Some(attr) = class_get_attr(&mcl, &name) { @@ -241,7 +241,7 @@ pub fn issubclass(subclass: &PyClassRef, cls: &PyClassRef) -> bool { } pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - debug!("type.__new__ {:?}", args); + vm_trace!("type.__new__ {:?}", args); if args.args.len() == 2 { Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { @@ -265,7 +265,7 @@ pub fn type_new_class( } pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { - debug!("type_call: {:?}", class); + vm_trace!("type_call: {:?}", class); let new = class_get_attr(&class, "__new__").expect("All types should have a __new__."); let new_wrapped = vm.call_get_descriptor(new, class.into_object())?; let obj = vm.invoke(new_wrapped, (&args, &kwargs))?; @@ -353,7 +353,7 @@ fn take_next_base(mut bases: Vec>) -> Option<(PyClassRef, Vec>) -> Option> { - debug!("Linearising MRO: {:?}", bases); + vm_trace!("Linearising MRO: {:?}", bases); let mut result = vec![]; loop { if (&bases).iter().all(Vec::is_empty) { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index f117b83f3..6e11dbd3d 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -170,7 +170,7 @@ impl VirtualMachine { fn new_exception_obj(&self, exc_type: PyClassRef, args: Vec) -> PyResult { // TODO: add repr of args into logging? - info!("New exception created: {}", exc_type.name); + vm_trace!("New exception created: {}", exc_type.name); self.invoke(exc_type.into_object(), args) } @@ -377,7 +377,7 @@ impl VirtualMachine { let cls = obj.class(); match objtype::class_get_attr(&cls, method_name) { Some(func) => { - trace!( + vm_trace!( "vm.call_method {:?} {:?} {:?} -> {:?}", obj, cls, @@ -392,7 +392,8 @@ impl VirtualMachine { } fn _invoke(&self, func_ref: PyObjectRef, args: PyFuncArgs) -> PyResult { - trace!("Invoke: {:?} {:?}", func_ref, args); + vm_trace!("Invoke: {:?} {:?}", func_ref, args); + if let Some(PyFunction { ref code, ref scope, @@ -411,7 +412,7 @@ impl VirtualMachine { value(self, args) } else { // TODO: is it safe to just invoke __call__ otherwise? - trace!("invoke __call__ for: {:?}", &func_ref.payload); + vm_trace!("invoke __call__ for: {:?}", &func_ref.payload); self.call_method(&func_ref, "__call__", args) } } @@ -630,7 +631,7 @@ impl VirtualMachine { T: TryIntoRef, { let attr_name = attr_name.try_into_ref(self)?; - trace!("vm.__getattribute__: {:?} {:?}", obj, attr_name); + vm_trace!("vm.__getattribute__: {:?} {:?}", obj, attr_name); self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } From 3d6673cf692d2ec119582bfb38775c3f33bd391a Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 7 Jul 2019 19:45:36 +0200 Subject: [PATCH 2/2] Formatting of macros.rs --- vm/src/macros.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 158898c48..617729674 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -257,7 +257,6 @@ macro_rules! match_class { }; } - /// Super detailed logging. Might soon overflow your logbuffers /// Default, this logging is discarded, except when a the `vm-tracing-logging` /// build feature is enabled.