mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Try to repair cpython bytecode frontend

Browse Source
This commit is contained in:
Windel Bouwman committed 2018-07-08 00:22:30 +02:00
1 parent 3f98e3c64a
commit b4e9064516
6 files changed
+83 -293
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
-3
View File
@@ -548,9 +548,6 @@ version = "0.1.0"
dependencies = [
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
py_code_object/Cargo.lock
Generated
+9
View File
@@ -87,6 +87,7 @@ version = "0.1.0"
dependencies = [
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"rustpython_vm 0.1.0",
"serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -114,6 +115,14 @@ name = "regex-syntax"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rustpython_vm"
version = "0.1.0"
dependencies = [
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde"
version = "0.8.23"
py_code_object/src/convert.rs
+61 -285
View File
@@ -3,249 +3,97 @@
// this into RustPython bytecode. This to decouple RustPython from CPython
// internal bytecode representations.
use super::bytecode::{self, CodeObject, Instruction};
use rustpython_vm::bytecode;
use py_code_object::{PyCodeObject, NativeType};
pub fn convert(cpython_bytecode: CPythonByteCode) -> ByteCode {
let c = Converter::new();
c.convert(c)
pub fn convert(cpython_bytecode: PyCodeObject) -> bytecode::CodeObject {
let mut c = Converter::new();
c.convert(cpython_bytecode);
c.code2
}
// TODO: think of an appropriate name for this thing:
pub struct Converter {
frames: Vec<Frame>,
code: Option<PyCodeObject>,
code2: bytecode::CodeObject,
}
impl Converter {
pub fn new() -> Converter {
Converter {
frames: vec![],
code: None,
code2: bytecode::CodeObject::new(),
}
}
fn curr_frame(&mut self) -> &mut Frame {
self.frames.last_mut().unwrap()
}
fn pop_frame(&mut self) {
self.frames.pop().unwrap();
}
fn unwind(&mut self, reason: String) {
let curr_frame = self.curr_frame();
let curr_block = curr_frame.blocks[curr_frame.blocks.len()-1].clone(); // use last?
curr_frame.why = reason; // Why do we need this?
debug!("block status: {:?}, {:?}", curr_block.block_type, curr_frame.why);
match (curr_block.block_type.as_ref(), curr_frame.why.as_ref()) {
("loop", "break") => {
curr_frame.lasti = curr_block.handler; //curr_frame.labels[curr_block.handler]; // Jump to the end
// Return the why as None
curr_frame.blocks.pop();
},
("loop", "none") => (), //skipped
_ => panic!("block stack operation not implemented")
pub fn convert(&mut self, code: PyCodeObject) {
self.code = Some(code);
for op_code in self.code.unwrap().co_code {
self.dispatch(op_code);
}
}
// Can we get rid of the code paramter?
fn make_frame(&self, code: PyCodeObject, callargs: HashMap<String, Rc<NativeType>>, globals: Option<HashMap<String, Rc<NativeType>>>) -> Frame {
//populate the globals and locals
let mut labels = HashMap::new();
let mut curr_offset = 0;
for (idx, op) in code.co_code.iter().enumerate() {
labels.insert(curr_offset, idx);
curr_offset += op.0;
}
//TODO: This is wrong, check https://github.com/nedbat/byterun/blob/31e6c4a8212c35b5157919abff43a7daa0f377c6/byterun/pyvm2.py#L95
let globals = match globals {
Some(g) => g,
None => HashMap::new(),
};
let mut locals = globals;
locals.extend(callargs);
//TODO: move this into the __builtin__ module when we have a module type
locals.insert("print".to_string(), Rc::new(NativeType::NativeFunction(builtins::print)));
locals.insert("len".to_string(), Rc::new(NativeType::NativeFunction(builtins::len)));
Frame {
code: code,
stack: vec![],
blocks: vec![],
// save the callargs as locals
globals: locals.clone(),
locals: locals,
labels: labels,
lasti: 0,
return_value: NativeType::NoneType,
why: "none".to_string(),
}
}
// The Option<i32> is the return value of the frame, remove when we have implemented frame
// TODO: read the op codes directly from the internal code object
fn run_frame(&mut self, frame: Frame) -> NativeType {
self.frames.push(frame);
//let mut why = None;
// Change this to a loop for jump
loop {
//while curr_frame.lasti < curr_frame.code.co_code.len() {
let op_code = {
let curr_frame = self.curr_frame();
if curr_frame.code.co_code.len() == 0 { panic!("Trying to run an empty frame. Check if the bytecode is empty"); }
let op_code = curr_frame.code.co_code[curr_frame.lasti].clone();
curr_frame.lasti += 1;
op_code
};
let why = self.dispatch(op_code);
/*if curr_frame.blocks.len() > 0 {
self.manage_block_stack(&why);
}
*/
if let Some(_) = why {
break;
}
}
let return_value = {
//let curr_frame = self.frames.last_mut().unwrap();
self.curr_frame().return_value.clone()
};
self.pop_frame();
return_value
}
pub fn run_code(&mut self, code: PyCodeObject) {
let frame = self.make_frame(code, HashMap::new(), None);
self.run_frame(frame);
// check if there are any leftover frame, fail if any
}
fn dispatch(&mut self, op_code: (usize, String, Option<usize>)) {
{
debug!("stack:{:?}", self.curr_frame().stack);
debug!("env :{:?}", self.curr_frame().locals);
debug!("Executing op code: {:?}", op_code);
}
debug!("Converting op code: {:?}", op_code);
match (op_code.1.as_ref(), op_code.2) {
("LOAD_CONST", Some(consti)) => {
// println!("Loading const at index: {}", consti);
let curr_frame = self.curr_frame();
let value = curr_frame.code.co_consts[consti].clone();
self.emit(LoadConst(None));
let value = match self.code.unwrap().co_consts[consti] {
// NativeType::Boolean { value } => { bytecode::Constant::Boolean { true } },
// NativeType::Int { value } => { bytecode::Constant::Integer { value: value } },
_ => { panic!("Not impl "); }
};
self.emit(bytecode::Instruction::LoadConst { value: value });
},
// TODO: universal stack element type
("LOAD_CONST", None) => {
self.emit(LoadConst(None));
self.emit(bytecode::Instruction::LoadConst { value: bytecode::Constant::None });
},
("POP_TOP", None) => {
self.emit(Pop(None));
},
("LOAD_FAST", Some(var_num)) => {
let curr_frame = self.curr_frame();
let ref name = curr_frame.code.co_varnames[var_num];
curr_frame.stack.push(curr_frame.locals.get::<str>(name).unwrap().clone());
self.emit(bytecode::Instruction::Pop);
},
("STORE_NAME", Some(namei)) => {
// println!("Loading const at index: {}", consti);
let curr_frame = self.curr_frame();
curr_frame.locals.insert(curr_frame.code.co_names[namei].clone(), curr_frame.stack.pop().unwrap().clone());
let name = self.code.unwrap().co_names[namei].clone();
self.emit(bytecode::Instruction::StoreName { name });
},
("LOAD_NAME", Some(namei)) => {
// println!("Loading const at index: {}", consti);
let curr_frame = self.curr_frame();
&curr_frame.code.co_names[namei]
let name = code.clone();
self.emit(Instruction::LoadName { name });
let name = self.code.unwrap().co_names[namei].clone();
self.emit(bytecode::Instruction::LoadName { name });
},
("LOAD_GLOBAL", Some(namei)) => {
// We need to load the underlying value the name points to, but stuff like
// AssertionError is in the names right after compile, so we load the string
// instead for now
let curr_frame = self.curr_frame();
let name = &curr_frame.code.co_names[namei];
curr_frame.stack.push(curr_frame.globals.get::<str>(name).unwrap().clone());
// let curr_frame = self.curr_frame();
// let name = &curr_frame.code.co_names[namei];
},
("BUILD_LIST", Some(count)) => {
self.emit(Instruction::BuildList { size: count });
self.emit(bytecode::Instruction::BuildList { size: count });
},
/*
("BUILD_SLICE", Some(count)) => {
let curr_frame = self.curr_frame();
assert!(count == 2 || count == 3);
let mut vec = vec!();
for _ in 0..count {
vec.push(curr_frame.stack.pop().unwrap());
}
vec.reverse();
let mut out:Vec<Option<i32>> = vec.into_iter().map(|x| match *x {
NativeType::Int(n) => Some(n),
NativeType::NoneType => None,
_ => panic!("Expect Int or None as BUILD_SLICE arguments, got {:?}", x),
}).collect();
if out.len() == 2 {
out.push(None);
}
assert!(out.len() == 3);
// TODO: assert the stop start and step are NativeType::Int
// See https://users.rust-lang.org/t/how-do-you-assert-enums/1187/8
curr_frame.stack.push(Rc::new(NativeType::Slice(out[0], out[1], out[2])));
None
},
*/
("GET_ITER", None) => {
self.emit(Instruction::GetIter);
self.emit(bytecode::Instruction::GetIter);
},
("FOR_ITER", Some(delta)) => {
self.emit(Instruction::ForIter);
self.emit(bytecode::Instruction::ForIter);
},
/*
("COMPARE_OP", Some(cmp_op_i)) => {
let curr_frame = self.curr_frame();
let v1 = curr_frame.stack.pop().unwrap();
let v2 = curr_frame.stack.pop().unwrap();
match CMP_OP[cmp_op_i] {
// To avoid branch explotion, use an array of callables instead
"==" => {
match (v1.deref(), v2.deref()) {
(&NativeType::Int(ref v1i), &NativeType::Int(ref v2i)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2i == v1i)));
},
(&NativeType::Float(ref v1f), &NativeType::Float(ref v2f)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == v1f)));
},
(&NativeType::Str(ref v1s), &NativeType::Str(ref v2s)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2s == v1s)));
},
(&NativeType::Int(ref v1i), &NativeType::Float(ref v2f)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == &(*v1i as f64))));
},
(&NativeType::List(ref l1), &NativeType::List(ref l2)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(l2 == l1)));
},
_ => panic!("TypeError in COMPARE_OP: can't compare {:?} with {:?}", v1, v2)
};
}
">" => {
match (v1.deref(), v2.deref()) {
(&NativeType::Int(ref v1i), &NativeType::Int(ref v2i)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2i < v1i)));
},
(&NativeType::Float(ref v1f), &NativeType::Float(ref v2f)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f < v1f)));
},
_ => panic!("TypeError in COMPARE_OP")
};
}
_ => panic!("Unimplemented COMPARE_OP operator")
}
None
},
("POP_JUMP_IF_TRUE", Some(ref target)) => {
let curr_frame = self.curr_frame();
@@ -292,16 +140,7 @@ impl Converter {
panic!("{:?}", exception);
}
("INPLACE_ADD", None) => {
let curr_frame = self.curr_frame();
let tos = curr_frame.stack.pop().unwrap();
let tos1 = curr_frame.stack.pop().unwrap();
match (tos.deref(), tos1.deref()) {
(&NativeType::Int(ref tosi), &NativeType::Int(ref tos1i)) => {
curr_frame.stack.push(Rc::new(NativeType::Int(tos1i + tosi)));
},
_ => panic!("TypeError in BINARY_ADD")
}
None
self.emit(bytecode::Instruction::BinaryOperation { op: BinaryOperator::Add });
},
("STORE_SUBSCR", None) => {
@@ -320,125 +159,64 @@ impl Converter {
_ => panic!("TypeError in STORE_SUBSCR")
}
curr_frame.stack.push(tos1);
None
},
*/
("BINARY_ADD", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::Add });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Add });
},
("BINARY_POWER", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::Power });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Power });
},
("BINARY_MULTIPLY", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::Multiply });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Multiply });
},
("BINARY_TRUE_DIVIDE", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::TrueDivide });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Divide });
},
("BINARY_MODULO", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::Modulo });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Modulo });
},
("BINARY_SUBTRACT", None) => {
self.emit(Instruction::BinaryOperation { op: BinaryOperator::Subtract });
self.emit(bytecode::Instruction::BinaryOperation { op: bytecode::BinaryOperator::Subtract });
},
/*
("ROT_TWO", None) => {
let curr_frame = self.curr_frame();
let tos = curr_frame.stack.pop().unwrap();
let tos1 = curr_frame.stack.pop().unwrap();
curr_frame.stack.push(tos);
curr_frame.stack.push(tos1);
None
// TODO: self.emit(Instruction::BinaryOperation { op: bytecode::BinaryOperator::RotTwo });
}
*/
("UNARY_NEGATIVE", None) => {
self.emit(Instruction::UnaryOperation { op: UnaryOperator::Minus });
self.emit(bytecode::Instruction::UnaryOperation { op: bytecode::UnaryOperator::Minus });
},
("UNARY_POSITIVE", None) => {
self.emit(Instruction::UnaryOperation { op: UnaryOperator::Plus });
self.emit(bytecode::Instruction::UnaryOperation { op: bytecode::UnaryOperator::Plus });
},
/*
("PRINT_ITEM", None) => {
let curr_frame = self.curr_frame();
// TODO: Print without the (...)
println!("{:?}", curr_frame.stack.pop().unwrap());
},
("PRINT_NEWLINE", None) => {
print!("\n");
},
},*/
/*
("MAKE_FUNCTION", Some(argc)) => {
// https://docs.python.org/3.4/library/dis.html#opcode-MAKE_FUNCTION
let curr_frame = self.curr_frame();
let qualified_name = curr_frame.stack.pop().unwrap();
let code_obj = match curr_frame.stack.pop().unwrap().deref() {
&NativeType::Code(ref code) => code.clone(),
_ => panic!("Second item on the stack should be a code object")
};
// pop argc arguments
// argument: name, args, globals
let func = Function::new(code_obj);
curr_frame.stack.push(Rc::new(NativeType::Function(func)));
None
self.emit(bytecode::Instruction::MakeFunction { });
},
*/
("CALL_FUNCTION", Some(argc)) => {
let kw_count = (argc >> 8) as u8;
let pos_count = (argc & 0xFF) as u8;
// Pop the arguments based on argc
let mut kw_args = HashMap::new();
let mut pos_args = Vec::new();
{
let curr_frame = self.curr_frame();
for _ in 0..kw_count {
let native_val = curr_frame.stack.pop().unwrap();
let native_key = curr_frame.stack.pop().unwrap();
if let (ref val, &NativeType::Str(ref key)) = (native_val, native_key.deref()) {
kw_args.insert(key.clone(), val.clone());
}
else {
panic!("Incorrect type found while building keyword argument list")
}
}
for _ in 0..pos_count {
pos_args.push(curr_frame.stack.pop().unwrap());
}
}
let locals = {
// FIXME: no clone here
self.curr_frame().locals.clone()
};
let func = {
match self.curr_frame().stack.pop().unwrap().deref() {
&NativeType::Function(ref func) => {
// pop argc arguments
// argument: name, args, globals
// build the callargs hashmap
pos_args.reverse();
let mut callargs = HashMap::new();
for (name, val) in func.code.co_varnames.iter().zip(pos_args) {
callargs.insert(name.to_string(), val);
}
// merge callargs with kw_args
let return_value = {
let frame = self.make_frame(func.code.clone(), callargs, Some(locals));
self.run_frame(frame)
};
self.curr_frame().stack.push(Rc::new(return_value));
},
&NativeType::NativeFunction(func) => {
pos_args.reverse();
let return_value = func(pos_args);
self.curr_frame().stack.push(Rc::new(return_value));
},
_ => panic!("The item on the stack should be a code object")
}
};
None
let pos_count = (argc & 0xFF) as usize;
self.emit(bytecode::Instruction::CallFunction { count: pos_count });
},
/*
("RETURN_VALUE", None) => {
// Hmmm... what is this used?
// I believe we need to push this to the next frame
self.curr_frame().return_value = (*self.curr_frame().stack.pop().unwrap()).clone();
Some("return".to_string())
self.emit(bytecode::Instruction::CallFunction { });
},
("SETUP_LOOP", Some(delta)) => {
let curr_frame = self.curr_frame();
@@ -447,15 +225,13 @@ impl Converter {
block_type: "loop".to_string(),
handler: *curr_frame.labels.get(&(curr_offset + delta)).unwrap(),
});
None
},
*/
("POP_BLOCK", None) => {
self.curr_frame().blocks.pop();
None
self.emit(bytecode::Instruction::PopBlock);
}
("SetLineno", _) | ("LABEL", _)=> {
// Skip
None
},
(name, _) => {
panic!("Unrecongnizable op code: {}", name);
@@ -463,7 +239,7 @@ impl Converter {
} // end match
} // end dispatch function
fn emit(&mut self, instruction: Instruction) {
self.code_object.instructions.push(instruction);
fn emit(&mut self, instruction: bytecode::Instruction) {
self.code2.instructions.push(instruction);
}
}
py_code_object/src/main.rs
+10 -5
View File
@@ -3,7 +3,12 @@ extern crate py_code_object;
extern crate rustpython_vm;
extern crate serde_json;
use rustpython_vm::*;
#[macro_use]
extern crate log;
mod convert;
use rustpython_vm::evaluate;
use std::env;
use std::fs::File;
use std::io::prelude::*;
@@ -21,12 +26,12 @@ fn main() {
f.read_to_string(&mut s).unwrap();
// println!("Read string");
// TODO: Extract this so we don't depend on json
let code: PyCodeObject = match serde_json::from_str(&s) {
let cpython_code: PyCodeObject = match serde_json::from_str(&s) {
Ok(c) => c,
Err(_) => panic!("Fail to parse the bytecode")
};
let mut vm = VirtualMachine::new();
vm.run_code(code);
// println!("Done");
let code = convert::convert(cpython_code);
evaluate(code);
}
vm/src/bytecode.rs
+2
View File
@@ -59,6 +59,7 @@ pub enum Constant {
Integer { value: i32 }, // TODO: replace by arbitrary big int math.
// TODO: Float { value: f64 },
String { value: String },
None,
}
#[derive(Debug, Clone)]
@@ -82,6 +83,7 @@ pub enum BinaryOperator {
pub enum UnaryOperator {
Not,
Minus,
Plus,
}
/*
vm/src/vm.rs
+1
View File
@@ -171,6 +171,7 @@ impl VirtualMachine {
&bytecode::Constant::Integer { ref value } => { PyObject::Integer { value: *value } },
// &bytecode::Constant::Float
&bytecode::Constant::String { ref value } => { PyObject::String { value: value.clone() } },
&bytecode::Constant::None => { PyObject::None },
};
frame.stack.push(obj.into_ref().clone());
},