diff --git a/Cargo.lock b/Cargo.lock index e1389c688..3a2455e04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -344,6 +344,20 @@ dependencies = [ "bitflags", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static 1.4.0", + "libc", + "terminal_size", + "termios", + "winapi", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -562,6 +576,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + [[package]] name = "digest" version = "0.8.1" @@ -627,6 +647,12 @@ dependencies = [ "strsim 0.9.3", ] +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + [[package]] name = "either" version = "1.6.1" @@ -642,6 +668,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "env_logger" version = "0.7.1" @@ -853,6 +885,20 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "insta" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e7528a20113cf7ca90eddfc2439c608188b6eafc0613964da2bd140c92acec" +dependencies = [ + "console", + "difference", + "lazy_static 1.4.0", + "serde", + "serde_json", + "serde_yaml", +] + [[package]] name = "instant" version = "0.1.7" @@ -1021,6 +1067,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + [[package]] name = "lock_api" version = "0.4.1" @@ -1628,6 +1680,7 @@ version = "0.1.2" dependencies = [ "bincode", "bitflags", + "bstr", "itertools", "lz4-compression", "num-bigint", @@ -1660,6 +1713,7 @@ version = "0.1.2" dependencies = [ "arrayvec", "indexmap", + "insta", "itertools", "log", "num-complex", @@ -1929,6 +1983,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3e2dd40a7cdc18ca80db804b7f461a39bb721160a85c9a1fa30134bf3c02a5" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -2088,6 +2154,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -2524,3 +2609,12 @@ checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi", ] + +[[package]] +name = "yaml-rust" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" +dependencies = [ + "linked-hash-map", +] diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml index 2188e8d18..c3c661155 100644 --- a/bytecode/Cargo.toml +++ b/bytecode/Cargo.toml @@ -16,3 +16,4 @@ num-bigint = { version = "0.3", features = ["serde"] } num-complex = { version = "0.3", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } itertools = "0.9" +bstr = "0.2" diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 6f3ebc604..371165739 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -2,11 +2,12 @@ //! implements bytecode structure. use bitflags::bitflags; +use bstr::ByteSlice; use itertools::Itertools; use num_bigint::BigInt; use num_complex::Complex64; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::fmt; /// Sourcecode location. @@ -30,13 +31,65 @@ impl Location { } } +pub trait Constant: Sized { + fn borrow_constant(&self) -> BorrowedConstant; + fn into_data(self) -> ConstantData { + self.borrow_constant().into_data() + } + fn map_constant(self, bag: &Bag) -> Bag::Constant { + bag.make_constant(self.into_data()) + } +} +impl Constant for ConstantData { + fn borrow_constant(&self) -> BorrowedConstant { + use BorrowedConstant::*; + match self { + ConstantData::Integer { value } => Integer { value }, + ConstantData::Float { value } => Float { value: *value }, + ConstantData::Complex { value } => Complex { value: *value }, + ConstantData::Boolean { value } => Boolean { value: *value }, + ConstantData::Str { value } => Str { value }, + ConstantData::Bytes { value } => Bytes { value }, + ConstantData::Code { code } => Code { code }, + ConstantData::Tuple { elements } => Tuple { + elements: Box::new(elements.iter().map(|e| e.borrow_constant())), + }, + ConstantData::None => None, + ConstantData::Ellipsis => Ellipsis, + } + } + fn into_data(self) -> ConstantData { + self + } +} + +pub trait ConstantBag: Sized { + type Constant: Constant; + fn make_constant(&self, constant: ConstantData) -> Self::Constant; + fn make_constant_borrowed(&self, constant: BorrowedConstant) -> Self::Constant { + self.make_constant(constant.into_data()) + } +} + +#[derive(Clone)] +pub struct BasicBag; +impl ConstantBag for BasicBag { + type Constant = ConstantData; + fn make_constant(&self, constant: ConstantData) -> Self::Constant { + constant + } + fn make_constant_borrowed(&self, constant: BorrowedConstant) -> Self::Constant { + constant.into_data() + } +} + /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. #[derive(Clone, PartialEq, Serialize, Deserialize)] -pub struct CodeObject { +pub struct CodeObject { pub instructions: Vec, /// Jump targets. - pub label_map: HashMap, + pub label_map: BTreeMap, pub locations: Vec, pub flags: CodeFlags, pub posonlyarg_count: usize, // Number of positional-only arguments @@ -47,6 +100,7 @@ pub struct CodeObject { pub source_path: String, pub first_line_number: usize, pub obj_name: String, // Name of the object that created this code object + pub constants: Vec, } bitflags! { @@ -82,7 +136,7 @@ impl CodeFlags { ]; } -#[derive(Serialize, Debug, Deserialize, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Serialize, Debug, Deserialize, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct Label(usize); impl Label { @@ -151,7 +205,8 @@ pub enum Instruction { name: String, }, LoadConst { - value: Constant, + /// index into constants vec + idx: usize, }, UnaryOperation { op: UnaryOperator, @@ -317,19 +372,89 @@ pub enum CallType { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum Constant { +pub enum ConstantData { Integer { value: BigInt }, Float { value: f64 }, Complex { value: Complex64 }, Boolean { value: bool }, - String { value: String }, + Str { value: String }, Bytes { value: Vec }, Code { code: Box }, - Tuple { elements: Vec }, + Tuple { elements: Vec }, None, Ellipsis, } +pub enum BorrowedConstant<'a, C: Constant> { + Integer { value: &'a BigInt }, + Float { value: f64 }, + Complex { value: Complex64 }, + Boolean { value: bool }, + Str { value: &'a str }, + Bytes { value: &'a [u8] }, + Code { code: &'a CodeObject }, + Tuple { elements: BorrowedTupleIter<'a, C> }, + None, + Ellipsis, +} +type BorrowedTupleIter<'a, C> = Box> + 'a>; +impl BorrowedConstant<'_, C> { + // takes `self` because we need to consume the iterator + pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + BorrowedConstant::Integer { value } => write!(f, "{}", value), + BorrowedConstant::Float { value } => write!(f, "{}", value), + BorrowedConstant::Complex { value } => write!(f, "{}", value), + BorrowedConstant::Boolean { value } => { + write!(f, "{}", if value { "True" } else { "False" }) + } + BorrowedConstant::Str { value } => write!(f, "{:?}", value), + BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()), + BorrowedConstant::Code { code } => write!(f, "{:?}", code), + BorrowedConstant::Tuple { elements } => { + write!(f, "(")?; + let mut first = true; + for c in elements { + if first { + first = false + } else { + write!(f, ", ")?; + } + c.fmt_display(f)?; + } + write!(f, ")") + } + BorrowedConstant::None => write!(f, "None"), + BorrowedConstant::Ellipsis => write!(f, "..."), + } + } + pub fn into_data(self) -> ConstantData { + use ConstantData::*; + match self { + BorrowedConstant::Integer { value } => Integer { + value: value.clone(), + }, + BorrowedConstant::Float { value } => Float { value }, + BorrowedConstant::Complex { value } => Complex { value }, + BorrowedConstant::Boolean { value } => Boolean { value }, + BorrowedConstant::Str { value } => Str { + value: value.to_owned(), + }, + BorrowedConstant::Bytes { value } => Bytes { + value: value.to_owned(), + }, + BorrowedConstant::Code { code } => Code { + code: Box::new(code.map_clone_bag(&BasicBag)), + }, + BorrowedConstant::Tuple { elements } => Tuple { + elements: elements.map(BorrowedConstant::into_data).collect(), + }, + BorrowedConstant::None => None, + BorrowedConstant::Ellipsis => Ellipsis, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum ComparisonOperator { Greater, @@ -378,7 +503,7 @@ pub enum BlockType { } */ -impl CodeObject { +impl CodeObject { #[allow(clippy::too_many_arguments)] pub fn new( flags: CodeFlags, @@ -390,10 +515,10 @@ impl CodeObject { source_path: String, first_line_number: usize, obj_name: String, - ) -> CodeObject { + ) -> Self { CodeObject { instructions: Vec::new(), - label_map: HashMap::new(), + label_map: BTreeMap::new(), locations: Vec::new(), flags, posonlyarg_count, @@ -404,32 +529,10 @@ impl CodeObject { source_path, first_line_number, obj_name, + constants: Vec::new(), } } - /// Load a code object from bytes - pub fn from_bytes(data: &[u8]) -> Result> { - let data = lz4_compression::decompress::decompress(data) - .map_err(|e| format!("lz4 error: {:?}", e))?; - bincode::deserialize::(&data).map_err(|e| e.into()) - } - - /// Serialize this bytecode to bytes. - pub fn to_bytes(&self) -> Vec { - let data = bincode::serialize(&self).expect("Code object must be serializable"); - lz4_compression::compress::compress(&data) - } - - pub fn get_constants(&self) -> impl Iterator { - self.instructions.iter().filter_map(|x| { - if let Instruction::LoadConst { value } = x { - Some(value) - } else { - None - } - }) - } - pub fn varnames(&self) -> impl Iterator + '_ { self.arg_names .iter() @@ -472,32 +575,96 @@ impl CodeObject { write!(f, " ")?; } write!(f, "{} {:5} ", arrow, offset)?; - instruction.fmt_dis(f, &self.label_map, expand_codeobjects, level)?; + instruction.fmt_dis( + f, + &self.label_map, + &self.constants, + expand_codeobjects, + level, + )?; } Ok(()) } pub fn display_expand_codeobjects<'a>(&'a self) -> impl fmt::Display + 'a { - struct Display<'a>(&'a CodeObject); - impl fmt::Display for Display<'_> { + struct Display<'a, C: Constant>(&'a CodeObject); + impl fmt::Display for Display<'_, C> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.0.display_inner(f, true, 1) } } Display(self) } + + pub fn map_bag(self, bag: &Bag) -> CodeObject { + CodeObject { + constants: self + .constants + .into_iter() + .map(|x| x.map_constant(bag)) + .collect(), + + instructions: self.instructions, + label_map: self.label_map, + locations: self.locations, + flags: self.flags, + posonlyarg_count: self.posonlyarg_count, + arg_names: self.arg_names, + varargs_name: self.varargs_name, + kwonlyarg_names: self.kwonlyarg_names, + varkeywords_name: self.varkeywords_name, + source_path: self.source_path, + first_line_number: self.first_line_number, + obj_name: self.obj_name, + } + } + + pub fn map_clone_bag(&self, bag: &Bag) -> CodeObject { + CodeObject { + constants: self + .constants + .iter() + .map(|x| bag.make_constant_borrowed(x.borrow_constant())) + .collect(), + + instructions: self.instructions.clone(), + label_map: self.label_map.clone(), + locations: self.locations.clone(), + flags: self.flags, + posonlyarg_count: self.posonlyarg_count, + arg_names: self.arg_names.clone(), + varargs_name: self.varargs_name.clone(), + kwonlyarg_names: self.kwonlyarg_names.clone(), + varkeywords_name: self.varkeywords_name.clone(), + source_path: self.source_path.clone(), + first_line_number: self.first_line_number, + obj_name: self.obj_name.clone(), + } + } } -impl fmt::Display for CodeObject { +impl CodeObject { + /// Load a code object from bytes + pub fn from_bytes(data: &[u8]) -> Result> { + let data = lz4_compression::decompress::decompress(data) + .map_err(|e| format!("lz4 error: {:?}", e))?; + bincode::deserialize(&data).map_err(|e| e.into()) + } + + /// Serialize this bytecode to bytes. + pub fn to_bytes(&self) -> Vec { + let data = bincode::serialize(&self).expect("Code object must be serializable"); + lz4_compression::compress::compress(&data) + } +} + +impl fmt::Display for CodeObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.display_inner(f, false, 1)?; - for constant in self.get_constants() { - match constant { - Constant::Code { code } => { - write!(f, "\nDisassembly of {}\n", constant)?; - code.fmt(f)?; - } - _ => continue, + for constant in &self.constants { + if let BorrowedConstant::Code { code } = constant.borrow_constant() { + write!(f, "\nDisassembly of {:?}\n", code)?; + code.fmt(f)?; } } Ok(()) @@ -505,27 +672,28 @@ impl fmt::Display for CodeObject { } impl Instruction { - fn fmt_dis( + fn fmt_dis( &self, f: &mut fmt::Formatter, - label_map: &HashMap, + label_map: &BTreeMap, + constants: &[C], expand_codeobjects: bool, level: usize, ) -> fmt::Result { macro_rules! w { ($variant:ident) => { - write!(f, "{:20}\n", stringify!($variant)) + writeln!(f, stringify!($variant)) }; ($variant:ident, $var:expr) => { - write!(f, "{:20} ({})\n", stringify!($variant), $var) + writeln!(f, "{:20} ({})", stringify!($variant), $var) }; ($variant:ident, $var1:expr, $var2:expr) => { - write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2) + writeln!(f, "{:20} ({}, {})", stringify!($variant), $var1, $var2) }; ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => { - write!( + writeln!( f, - "{:20} ({}, {}, {})\n", + "{:20} ({}, {}, {})", stringify!($variant), $var1, $var2, @@ -555,14 +723,21 @@ impl Instruction { DeleteSubscript => w!(DeleteSubscript), StoreAttr { name } => w!(StoreAttr, name), DeleteAttr { name } => w!(DeleteAttr, name), - LoadConst { value } => match value { - Constant::Code { code } if expand_codeobjects => { - writeln!(f, "LoadConst ({:?}):", code)?; - code.display_inner(f, true, level + 1)?; - Ok(()) + LoadConst { idx } => { + let value = &constants[*idx]; + match value.borrow_constant() { + BorrowedConstant::Code { code } if expand_codeobjects => { + writeln!(f, "{:20} ({:?}):", "LoadConst", code)?; + code.display_inner(f, true, level + 1)?; + Ok(()) + } + c => { + write!(f, "{:20} (", "LoadConst")?; + c.fmt_display(f)?; + writeln!(f, ")") + } } - _ => w!(LoadConst, value), - }, + } UnaryOperation { op } => w!(UnaryOperation, format!("{:?}", op)), BinaryOperation { op, inplace } => w!(BinaryOperation, format!("{:?}", op), inplace), LoadAttr { name } => w!(LoadAttr, name), @@ -625,32 +800,13 @@ impl Instruction { } } -impl fmt::Display for Constant { +impl fmt::Display for ConstantData { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Constant::Integer { value } => write!(f, "{}", value), - Constant::Float { value } => write!(f, "{}", value), - Constant::Complex { value } => write!(f, "{}", value), - Constant::Boolean { value } => write!(f, "{}", value), - Constant::String { value } => write!(f, "{:?}", value), - Constant::Bytes { value } => write!(f, "{:?}", value), - Constant::Code { code } => write!(f, "{:?}", code), - Constant::Tuple { elements } => write!( - f, - "({})", - elements - .iter() - .map(|e| format!("{}", e)) - .collect::>() - .join(", ") - ), - Constant::None => write!(f, "None"), - Constant::Ellipsis => write!(f, "Ellipsis"), - } + self.borrow_constant().fmt_display(f) } } -impl fmt::Debug for CodeObject { +impl fmt::Debug for CodeObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, @@ -661,7 +817,7 @@ impl fmt::Debug for CodeObject { } #[derive(Serialize, Deserialize)] -pub struct FrozenModule { - pub code: CodeObject, +pub struct FrozenModule { + pub code: CodeObject, pub package: bool, } diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 9c73674ee..6ca888182 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -15,3 +15,6 @@ rustpython-parser = { path = "../parser", version = "0.1.1" } num-complex = { version = "0.3", features = ["serde"] } log = "0.4" arrayvec = "0.5" + +[dev-dependencies] +insta = "1.1" diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 13eac5e22..4be7c307e 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -7,8 +7,6 @@ use crate::error::{CompileError, CompileErrorType}; pub use crate::mode::Mode; -use crate::output_stream::{CodeObjectStream, OutputStream}; -use crate::peephole::PeepholeOptimizer; use crate::symboltable::{ make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable, }; @@ -17,13 +15,11 @@ use num_complex::Complex64; use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label}; use rustpython_parser::{ast, parser}; -type BasicOutputStream = PeepholeOptimizer; - type CompileResult = Result; /// Main structure holding the state of compilation. -struct Compiler { - output_stack: Vec, +struct Compiler { + output_stack: Vec, symbol_table_stack: Vec, nxt_label: usize, source_path: String, @@ -143,7 +139,7 @@ pub fn compile_program_single( }) } -impl Compiler { +impl Compiler { fn new(opts: CompileOpts, source_path: String) -> Self { Compiler { output_stack: Vec::new(), @@ -174,7 +170,7 @@ impl Compiler { } fn push_output(&mut self, code: CodeObject) { - self.output_stack.push(code.into()); + self.output_stack.push(code); } fn push_new_code_object(&mut self, obj_name: String) { @@ -193,7 +189,7 @@ impl Compiler { } fn pop_code_object(&mut self) -> CodeObject { - self.output_stack.pop().unwrap().into() + self.output_stack.pop().unwrap() } fn compile_program( @@ -206,9 +202,7 @@ impl Compiler { let (statements, doc) = get_doc(&program.statements); if let Some(value) = doc { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value }, - }); + self.emit_constant(bytecode::ConstantData::Str { value }); self.emit(Instruction::StoreName { name: "__doc__".to_owned(), scope: bytecode::NameScope::Global, @@ -219,9 +213,7 @@ impl Compiler { assert_eq!(self.output_stack.len(), size_before); // Emit None at end: - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::ReturnValue); Ok(()) } @@ -255,9 +247,7 @@ impl Compiler { } if !emitted_return { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::ReturnValue); } @@ -453,9 +443,7 @@ impl Compiler { if is_async { self.emit(Instruction::BeforeAsyncWith); self.emit(Instruction::GetAwaitable); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::YieldFrom); self.emit(Instruction::SetupAsyncWith { end: end_label }); } else { @@ -486,9 +474,7 @@ impl Compiler { if is_async { self.emit(Instruction::GetAwaitable); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::YieldFrom); } @@ -589,7 +575,10 @@ impl Compiler { match value { Some(v) => { if self.ctx.func == FunctionContext::AsyncFunction - && self.current_output().is_generator() + && self + .current_code() + .flags + .contains(bytecode::CodeFlags::IS_GENERATOR) { return Err(self.error_loc( CompileErrorType::AsyncReturnValue, @@ -599,9 +588,7 @@ impl Compiler { self.compile_expression(v)?; } None => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); } } @@ -687,10 +674,8 @@ impl Compiler { let mut num_kw_only_defaults = 0; for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) { if let Some(default) = default { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: kw.arg.clone(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: kw.arg.clone(), }); self.compile_expression(default)?; num_kw_only_defaults += 1; @@ -912,9 +897,7 @@ impl Compiler { // the last instruction is a ReturnValue already, we don't need to emit it } _ => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::ReturnValue); } } @@ -928,10 +911,8 @@ impl Compiler { // Return annotation: if let Some(annotation) = returns { // key: - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "return".to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: "return".to_owned(), }); // value: self.compile_expression(annotation)?; @@ -940,10 +921,8 @@ impl Compiler { let mut visit_arg_annotation = |arg: &ast::Parameter| -> CompileResult<()> { if let Some(annotation) = &arg.annotation { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: arg.arg.to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: arg.arg.to_owned(), }); self.compile_expression(&annotation)?; num_annotations += 1; @@ -976,15 +955,11 @@ impl Compiler { code.flags |= bytecode::CodeFlags::IS_COROUTINE; } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Code { - code: Box::new(code), - }, + self.emit_constant(bytecode::ConstantData::Code { + code: Box::new(code), }); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: qualified_name, - }, + self.emit_constant(bytecode::ConstantData::Str { + value: qualified_name, }); // Turn code object into function object: @@ -1106,10 +1081,8 @@ impl Compiler { name: "__module__".to_owned(), scope: bytecode::NameScope::Free, }); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: qualified_name.clone(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: qualified_name.clone(), }); self.emit(Instruction::StoreName { name: "__qualname__".to_owned(), @@ -1125,33 +1098,25 @@ impl Compiler { self.emit(Instruction::SetupAnnotation); } self.compile_statements(new_body)?; - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::ReturnValue); let mut code = self.pop_code_object(); - code.flags &= !bytecode::CodeFlags::NEW_LOCALS; + code.flags.remove(bytecode::CodeFlags::NEW_LOCALS); self.leave_scope(); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Code { - code: Box::new(code), - }, + self.emit_constant(bytecode::ConstantData::Code { + code: Box::new(code), }); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: name.to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: name.to_owned(), }); // Turn code object into function object: self.emit(Instruction::MakeFunction); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: qualified_name, - }, + self.emit_constant(bytecode::ConstantData::Str { + value: qualified_name, }); for base in bases { @@ -1162,7 +1127,7 @@ impl Compiler { let mut kwarg_names = vec![]; for keyword in keywords { if let Some(name) = &keyword.name { - kwarg_names.push(bytecode::Constant::String { + kwarg_names.push(bytecode::ConstantData::Str { value: name.to_owned(), }); } else { @@ -1172,10 +1137,8 @@ impl Compiler { self.compile_expression(&keyword.value)?; } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Tuple { - elements: kwarg_names, - }, + self.emit_constant(bytecode::ConstantData::Tuple { + elements: kwarg_names, }); self.emit(Instruction::CallFunction { typ: CallType::Keyword(2 + keywords.len() + bases.len()), @@ -1199,11 +1162,9 @@ impl Compiler { // Duplicate top of stack (the function or class object) // Doc string value: - self.emit(Instruction::LoadConst { - value: match doc_str { - Some(doc) => bytecode::Constant::String { value: doc }, - None => bytecode::Constant::None, // set docstring None if not declared - }, + self.emit_constant(match doc_str { + Some(doc) => bytecode::ConstantData::Str { value: doc }, + None => bytecode::ConstantData::None, // set docstring None if not declared }); } @@ -1273,9 +1234,7 @@ impl Compiler { handler: check_asynciter_label, }); self.emit(Instruction::GetANext); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::YieldFrom); self.compile_store(target)?; self.emit(Instruction::PopBlock); @@ -1428,10 +1387,8 @@ impl Compiler { name: String::from("__annotations__"), scope: bytecode::NameScope::Local, }); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: name.to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: name.to_owned(), }); self.emit(Instruction::StoreSubscript); } else { @@ -1730,15 +1687,15 @@ impl Compiler { } Number { value } => { let const_value = match value { - ast::Number::Integer { value } => bytecode::Constant::Integer { + ast::Number::Integer { value } => bytecode::ConstantData::Integer { value: value.clone(), }, - ast::Number::Float { value } => bytecode::Constant::Float { value: *value }, - ast::Number::Complex { real, imag } => bytecode::Constant::Complex { + ast::Number::Float { value } => bytecode::ConstantData::Float { value: *value }, + ast::Number::Complex { real, imag } => bytecode::ConstantData::Complex { value: Complex64::new(*real, *imag), }, }; - self.emit(Instruction::LoadConst { value: const_value }); + self.emit_constant(const_value); } List { elements } => { let size = elements.len(); @@ -1781,9 +1738,7 @@ impl Compiler { self.mark_generator(); match value { Some(expression) => self.compile_expression(expression)?, - Option::None => self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }), + Option::None => self.emit_constant(bytecode::ConstantData::None), }; self.emit(Instruction::YieldValue); } @@ -1793,9 +1748,7 @@ impl Compiler { } self.compile_expression(value)?; self.emit(Instruction::GetAwaitable); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::YieldFrom); } YieldFrom { value } => { @@ -1811,39 +1764,27 @@ impl Compiler { self.mark_generator(); self.compile_expression(value)?; self.emit(Instruction::GetIter); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); self.emit(Instruction::YieldFrom); } True => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Boolean { value: true }, - }); + self.emit_constant(bytecode::ConstantData::Boolean { value: true }); } False => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Boolean { value: false }, - }); + self.emit_constant(bytecode::ConstantData::Boolean { value: false }); } ast::ExpressionType::None => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); + self.emit_constant(bytecode::ConstantData::None); } Ellipsis => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Ellipsis, - }); + self.emit_constant(bytecode::ConstantData::Ellipsis); } - String { value } => { + ast::ExpressionType::String { value } => { self.compile_string(value)?; } Bytes { value } => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Bytes { - value: value.clone(), - }, + self.emit_constant(bytecode::ConstantData::Bytes { + value: value.clone(), }); } Identifier { name } => { @@ -1862,14 +1803,10 @@ impl Compiler { self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); self.leave_scope(); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Code { - code: Box::new(code), - }, - }); - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: name }, + self.emit_constant(bytecode::ConstantData::Code { + code: Box::new(code), }); + self.emit_constant(bytecode::ConstantData::Str { value: name }); // Turn code object into function object: self.emit(Instruction::MakeFunction); @@ -1916,10 +1853,8 @@ impl Compiler { let mut subsize = 0; for keyword in subkeywords { if let Some(name) = &keyword.name { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: name.to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: name.to_owned(), }); self.compile_expression(&keyword.value)?; subsize += 1; @@ -1980,7 +1915,7 @@ impl Compiler { let mut kwarg_names = vec![]; for keyword in keywords { if let Some(name) = &keyword.name { - kwarg_names.push(bytecode::Constant::String { + kwarg_names.push(bytecode::ConstantData::Str { value: name.to_owned(), }); } else { @@ -1990,10 +1925,8 @@ impl Compiler { self.compile_expression(&keyword.value)?; } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Tuple { - elements: kwarg_names, - }, + self.emit_constant(bytecode::ConstantData::Tuple { + elements: kwarg_names, }); self.emit(Instruction::CallFunction { typ: CallType::Keyword(count), @@ -2189,16 +2122,12 @@ impl Compiler { self.leave_scope(); // List comprehension code: - self.emit(Instruction::LoadConst { - value: bytecode::Constant::Code { - code: Box::new(code), - }, + self.emit_constant(bytecode::ConstantData::Code { + code: Box::new(code), }); // List comprehension function name: - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: name }, - }); + self.emit_constant(bytecode::ConstantData::Str { value: name }); // Turn code object into function object: self.emit(Instruction::MakeFunction); @@ -2218,9 +2147,7 @@ impl Compiler { fn compile_string(&mut self, string: &ast::StringGroup) -> CompileResult<()> { if let Some(value) = try_get_constant_string(string) { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value }, - }); + self.emit_constant(bytecode::ConstantData::Str { value }); } else { match string { ast::StringGroup::Joined { values } => { @@ -2230,10 +2157,8 @@ impl Compiler { self.emit(Instruction::BuildString { size: values.len() }) } ast::StringGroup::Constant { value } => { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: value.to_owned(), - }, + self.emit_constant(bytecode::ConstantData::Str { + value: value.to_owned(), }); } ast::StringGroup::FormattedValue { @@ -2243,10 +2168,8 @@ impl Compiler { } => { match spec { Some(spec) => self.compile_string(spec)?, - None => self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: String::new(), - }, + None => self.emit_constant(bytecode::ConstantData::Str { + value: String::new(), }), }; self.compile_expression(value)?; @@ -2312,10 +2235,19 @@ impl Compiler { fn emit(&mut self, instruction: Instruction) { let location = compile_location(&self.current_source_location); // TODO: insert source filename - self.current_output().emit(instruction, location); + let code = self.current_code(); + code.instructions.push(instruction); + code.locations.push(location); } - fn current_output(&mut self) -> &mut O { + fn emit_constant(&mut self, constant: bytecode::ConstantData) { + let code = self.current_code(); + let idx = code.constants.len(); + code.constants.push(constant); + self.emit(Instruction::LoadConst { idx }) + } + + fn current_code(&mut self) -> &mut CodeObject { self.output_stack .last_mut() .expect("No OutputStream on stack") @@ -2330,7 +2262,9 @@ impl Compiler { // Assign current position the given label fn set_label(&mut self, label: Label) { - self.current_output().set_label(label) + let code = self.current_code(); + let pos = code.instructions.len(); + code.label_map.insert(label, pos); } fn set_source_location(&mut self, location: ast::Location) { @@ -2350,7 +2284,7 @@ impl Compiler { } fn mark_generator(&mut self) { - self.current_output().mark_generator(); + self.current_code().flags |= bytecode::CodeFlags::IS_GENERATOR } } @@ -2404,9 +2338,7 @@ fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::Co mod tests { use super::{CompileOpts, Compiler}; use crate::symboltable::make_symbol_table; - use rustpython_bytecode::bytecode::Constant::*; - use rustpython_bytecode::bytecode::Instruction::*; - use rustpython_bytecode::bytecode::{CodeObject, Label}; + use rustpython_bytecode::bytecode::CodeObject; use rustpython_parser::parser; fn compile_exec(source: &str) -> CodeObject { @@ -2419,119 +2351,43 @@ mod tests { compiler.pop_code_object() } + macro_rules! assert_dis_snapshot { + ($value:expr) => { + insta::assert_snapshot!( + insta::internals::AutoName, + $value.display_expand_codeobjects().to_string(), + stringify!($value) + ) + }; + } + #[test] fn test_if_ors() { - let code = compile_exec("if True or False or False:\n pass\n"); - assert_eq!( - vec![ - LoadConst { - value: Boolean { value: true } - }, - JumpIfTrue { - target: Label::new(1) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfTrue { - target: Label::new(1) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { value: None }, - ReturnValue - ], - code.instructions - ); + assert_dis_snapshot!(compile_exec( + "\ +if True or False or False: + pass +" + )); } #[test] fn test_if_ands() { - let code = compile_exec("if True and False and False:\n pass\n"); - assert_eq!( - vec![ - LoadConst { - value: Boolean { value: true } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { value: None }, - ReturnValue - ], - code.instructions - ); + assert_dis_snapshot!(compile_exec( + "\ +if True and False and False: + pass +" + )); } #[test] fn test_if_mixed() { - let code = compile_exec("if (True and False) or (False and True):\n pass\n"); - assert_eq!( - vec![ - LoadConst { - value: Boolean { value: true } - }, - JumpIfFalse { - target: Label::new(2) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfTrue { - target: Label::new(1) - }, - LoadConst { - value: Boolean { value: false } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { - value: Boolean { value: true } - }, - JumpIfFalse { - target: Label::new(0) - }, - LoadConst { value: None }, - ReturnValue - ], - code.instructions - ); - } - - #[test] - fn test_constant_optimization() { - let code = compile_exec("1 + 2 + 3 + 4\n1.5 * 2.5"); - assert_eq!( - code.instructions, - vec![ - LoadConst { - value: Integer { value: 10.into() } - }, - Pop, - LoadConst { - value: Float { value: 3.75 } - }, - Pop, - LoadConst { value: None }, - ReturnValue, - ] - ); + assert_dis_snapshot!(compile_exec( + "\ +if (True and False) or (False and True): + pass +" + )); } } diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 4d4693505..415ff6134 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -9,6 +9,4 @@ extern crate log; pub mod compile; pub mod error; pub mod mode; -pub(crate) mod output_stream; -pub mod peephole; pub mod symboltable; diff --git a/compiler/src/output_stream.rs b/compiler/src/output_stream.rs deleted file mode 100644 index 9cc49a944..000000000 --- a/compiler/src/output_stream.rs +++ /dev/null @@ -1,44 +0,0 @@ -use rustpython_bytecode::bytecode::{CodeFlags, CodeObject, Instruction, Label, Location}; - -pub trait OutputStream: From + Into { - /// Output an instruction - fn emit(&mut self, instruction: Instruction, location: Location); - /// Set a label on an instruction - fn set_label(&mut self, label: Label); - /// Mark the inner CodeObject as a generator - fn mark_generator(&mut self); - /// Check to see if the inner CodeObject is a generator - fn is_generator(&self) -> bool; -} - -pub struct CodeObjectStream { - code: CodeObject, -} - -impl From for CodeObjectStream { - fn from(code: CodeObject) -> Self { - CodeObjectStream { code } - } -} -impl From for CodeObject { - fn from(stream: CodeObjectStream) -> Self { - stream.code - } -} - -impl OutputStream for CodeObjectStream { - fn emit(&mut self, instruction: Instruction, location: Location) { - self.code.instructions.push(instruction); - self.code.locations.push(location); - } - fn set_label(&mut self, label: Label) { - let position = self.code.instructions.len(); - self.code.label_map.insert(label, position); - } - fn mark_generator(&mut self) { - self.code.flags |= CodeFlags::IS_GENERATOR; - } - fn is_generator(&self) -> bool { - self.code.flags.contains(CodeFlags::IS_GENERATOR) - } -} diff --git a/compiler/src/peephole.rs b/compiler/src/peephole.rs deleted file mode 100644 index aaec0470c..000000000 --- a/compiler/src/peephole.rs +++ /dev/null @@ -1,132 +0,0 @@ -use crate::output_stream::OutputStream; -use arrayvec::ArrayVec; -use rustpython_bytecode::bytecode::{CodeObject, Instruction, Label, Location}; - -pub mod optimizations; - -const PEEPHOLE_BUFFER_SIZE: usize = 20; - -pub struct InstructionMetadata { - loc: Location, - labels: Vec