From 6d1f2986feb56f635050fa98d44f2e5a22266792 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 15 Dec 2022 13:25:02 -0600 Subject: [PATCH 1/2] Custom marshal enc/decoding impl --- Cargo.lock | 18 - compiler/Cargo.toml | 2 - compiler/codegen/Cargo.toml | 3 +- compiler/codegen/src/compile.rs | 18 +- compiler/codegen/src/error.rs | 4 +- compiler/codegen/src/ir.rs | 18 +- compiler/core/Cargo.toml | 8 +- compiler/core/src/bytecode.rs | 408 +++++++++--------- compiler/core/src/error.rs | 18 +- compiler/core/src/lib.rs | 1 + compiler/core/src/location.rs | 4 +- compiler/core/src/marshal.rs | 615 ++++++++++++++++++++++++++++ compiler/parser/Cargo.toml | 1 - compiler/parser/src/parser.rs | 4 +- compiler/src/lib.rs | 38 +- vm/Cargo.toml | 7 +- vm/src/builtins/code.rs | 39 +- vm/src/builtins/complex.rs | 6 + vm/src/builtins/dict.rs | 59 ++- vm/src/builtins/float.rs | 1 + vm/src/builtins/function.rs | 14 +- vm/src/builtins/function/jitfunc.rs | 12 +- vm/src/builtins/traceback.rs | 1 + vm/src/exceptions.rs | 12 +- vm/src/frame.rs | 21 +- vm/src/lib.rs | 1 + vm/src/macros.rs | 4 +- vm/src/stdlib/marshal.rs | 467 +++++++-------------- wasm/lib/Cargo.toml | 2 +- 29 files changed, 1190 insertions(+), 616 deletions(-) create mode 100644 compiler/core/src/marshal.rs diff --git a/Cargo.lock b/Cargo.lock index 57721dc7d..f6c04b20e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,15 +126,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bit-set" version = "0.5.3" @@ -1390,7 +1381,6 @@ dependencies = [ "autocfg", "num-integer", "num-traits", - "serde", ] [[package]] @@ -1400,7 +1390,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" dependencies = [ "num-traits", - "serde", ] [[package]] @@ -1987,7 +1976,6 @@ dependencies = [ "rustpython-ast", "rustpython-compiler-core", "rustpython-parser", - "thiserror", ] [[package]] @@ -2022,23 +2010,18 @@ dependencies = [ "rustpython-codegen", "rustpython-compiler-core", "rustpython-parser", - "thiserror", ] [[package]] name = "rustpython-compiler-core" version = "0.2.0" dependencies = [ - "bincode", "bitflags", "bstr", "itertools", "lz4_flex", "num-bigint", "num-complex", - "num_enum", - "serde", - "thiserror", ] [[package]] @@ -2108,7 +2091,6 @@ dependencies = [ "rustc-hash", "rustpython-ast", "rustpython-compiler-core", - "thiserror", "tiny-keccak", "unic-emoji-char", "unic-ucd-ident", diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index b6e204dfd..ff6ea7750 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -9,5 +9,3 @@ edition = "2021" rustpython-compiler-core = { path = "core" } rustpython-codegen = { path = "codegen" } rustpython-parser = { path = "parser" } - -thiserror = { workspace = true } diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index fd6ad6533..3bb891ed3 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -16,9 +16,8 @@ bitflags = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } log = { workspace = true } -num-complex = { workspace = true, features = ["serde"] } +num-complex = { workspace = true } num-traits = { workspace = true } -thiserror = { workspace = true } [dev-dependencies] rustpython-parser = { path = "../parser" } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index be1c97db1..2e80de4ee 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -249,9 +249,9 @@ impl Compiler { fn push_output( &mut self, flags: bytecode::CodeFlags, - posonlyarg_count: usize, - arg_count: usize, - kwonlyarg_count: usize, + posonlyarg_count: u32, + arg_count: u32, + kwonlyarg_count: u32, obj_name: String, ) { let source_path = self.source_path.clone(); @@ -936,9 +936,11 @@ impl Compiler { self.push_output( bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, - args.posonlyargs.len(), - args.posonlyargs.len() + args.args.len(), - args.kwonlyargs.len(), + args.posonlyargs.len().try_into().unwrap(), + (args.posonlyargs.len() + args.args.len()) + .try_into() + .unwrap(), + args.kwonlyargs.len().try_into().unwrap(), name.to_owned(), ); @@ -2750,8 +2752,8 @@ impl Compiler { self.current_source_location = location; } - fn get_source_line_number(&self) -> usize { - self.current_source_location.row() + fn get_source_line_number(&self) -> u32 { + self.current_source_location.row() as u32 } fn push_qualified_path(&mut self, name: &str) { diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 648446f4f..e0380a62d 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -2,7 +2,7 @@ use std::fmt; pub type CodegenError = rustpython_compiler_core::BaseError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug)] #[non_exhaustive] pub enum CodegenErrorType { /// Invalid assignment, cannot store value in target. @@ -33,6 +33,8 @@ pub enum CodegenErrorType { NotImplementedYet, // RustPython marker for unimplemented features } +impl std::error::Error for CodegenErrorType {} + impl fmt::Display for CodegenErrorType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use CodegenErrorType::*; diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 43d7be0f2..2658028d1 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -63,11 +63,11 @@ impl Default for Block { pub struct CodeInfo { pub flags: CodeFlags, - pub posonlyarg_count: usize, // Number of positional-only arguments - pub arg_count: usize, - pub kwonlyarg_count: usize, + pub posonlyarg_count: u32, // Number of positional-only arguments + pub arg_count: u32, + pub kwonlyarg_count: u32, pub source_path: String, - pub first_line_number: usize, + pub first_line_number: u32, pub obj_name: String, // Name of the object that created this code object pub blocks: Vec, @@ -172,15 +172,15 @@ impl CodeInfo { } } - fn cell2arg(&self) -> Option> { + fn cell2arg(&self) -> Option> { if self.cellvar_cache.is_empty() { return None; } let total_args = self.arg_count + self.kwonlyarg_count - + self.flags.contains(CodeFlags::HAS_VARARGS) as usize - + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as usize; + + self.flags.contains(CodeFlags::HAS_VARARGS) as u32 + + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32; let mut found_cellarg = false; let cell2arg = self @@ -190,10 +190,10 @@ impl CodeInfo { self.varname_cache .get_index_of(var) // check that it's actually an arg - .filter(|i| *i < total_args) + .filter(|i| *i < total_args as usize) .map_or(-1, |i| { found_cellarg = true; - i as isize + i as i32 }) }) .collect::>(); diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index bf385a718..03160356d 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -8,14 +8,10 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [dependencies] -bincode = { workspace = true } bitflags = { workspace = true } bstr = { workspace = true } itertools = { workspace = true } -num-bigint = { workspace = true, features = ["serde"] } -num-complex = { workspace = true, features = ["serde"] } -num_enum = { workspace = true } -serde = { workspace = true, features = ["derive"] } -thiserror = { workspace = true } +num-bigint = { workspace = true } +num-complex = { workspace = true } lz4_flex = "0.9.2" diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index b7c456e98..fa7146d77 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -1,14 +1,12 @@ //! Implement python as a virtual machine with bytecodes. This module //! implements bytecode structure. -use crate::Location; +use crate::marshal::MarshalError; +use crate::{marshal, Location}; use bitflags::bitflags; -use bstr::ByteSlice; use itertools::Itertools; use num_bigint::BigInt; use num_complex::Complex64; -use num_enum::{IntoPrimitive, TryFromPrimitive}; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use std::{collections::BTreeSet, fmt, hash, mem}; @@ -31,9 +29,7 @@ impl Constant for ConstantData { ConstantData::Str { value } => Str { value }, ConstantData::Bytes { value } => Bytes { value }, ConstantData::Code { code } => Code { code }, - ConstantData::Tuple { elements } => Tuple { - elements: Box::new(elements.iter().map(|e| e.borrow_constant())), - }, + ConstantData::Tuple { elements } => Tuple { elements }, ConstantData::None => None, ConstantData::Ellipsis => Ellipsis, } @@ -44,6 +40,9 @@ impl Constant for ConstantData { pub trait ConstantBag: Sized + Copy { type Constant: Constant; fn make_constant(&self, constant: BorrowedConstant) -> Self::Constant; + fn make_int(&self, value: BigInt) -> Self::Constant; + fn make_tuple(&self, elements: impl Iterator) -> Self::Constant; + fn make_code(&self, code: CodeObject) -> Self::Constant; fn make_name(&self, name: &str) -> ::Name; } @@ -55,6 +54,19 @@ impl ConstantBag for BasicBag { fn make_constant(&self, constant: BorrowedConstant) -> Self::Constant { constant.to_owned() } + fn make_int(&self, value: BigInt) -> Self::Constant { + ConstantData::Integer { value } + } + fn make_tuple(&self, elements: impl Iterator) -> Self::Constant { + ConstantData::Tuple { + elements: elements.collect(), + } + } + fn make_code(&self, code: CodeObject) -> Self::Constant { + ConstantData::Code { + code: Box::new(code), + } + } fn make_name(&self, name: &str) -> ::Name { name.to_owned() } @@ -62,26 +74,22 @@ impl ConstantBag for BasicBag { /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone)] pub struct CodeObject { pub instructions: Box<[CodeUnit]>, pub locations: Box<[Location]>, pub flags: CodeFlags, - pub posonlyarg_count: usize, + pub posonlyarg_count: u32, // Number of positional-only arguments - pub arg_count: usize, - pub kwonlyarg_count: usize, + pub arg_count: u32, + pub kwonlyarg_count: u32, pub source_path: C::Name, - pub first_line_number: usize, + pub first_line_number: u32, pub max_stackdepth: u32, pub obj_name: C::Name, // Name of the object that created this code object - pub cell2arg: Option>, + pub cell2arg: Option>, pub constants: Box<[C]>, - #[serde(bound( - deserialize = "C::Name: serde::Deserialize<'de>", - serialize = "C::Name: serde::Serialize" - ))] pub names: Box<[C::Name]>, pub varnames: Box<[C::Name]>, pub cellvars: Box<[C::Name]>, @@ -89,7 +97,6 @@ pub struct CodeObject { } bitflags! { - #[derive(Serialize, Deserialize)] pub struct CodeFlags: u16 { const NEW_LOCALS = 0x01; const IS_GENERATOR = 0x02; @@ -114,7 +121,7 @@ impl CodeFlags { } /// an opcode argument that may be extended by a prior ExtendedArg -#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Copy, Clone, PartialEq, Eq)] #[repr(transparent)] pub struct OpArgByte(pub u8); impl OpArgByte { @@ -211,23 +218,7 @@ impl OpArgType for bool { } } -macro_rules! enum_oparg { - ($t:ident) => { - impl OpArgType for $t { - #[inline(always)] - fn from_oparg(x: u32) -> Option { - $t::try_from_primitive(x as _).ok() - } - #[inline(always)] - fn to_oparg(self) -> u32 { - u8::from(self).into() - } - } - }; -} - -#[derive(Copy, Clone, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Copy, Clone)] pub struct Arg(PhantomData); impl Arg { @@ -302,7 +293,7 @@ impl fmt::Display for Label { } /// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[repr(u8)] pub enum ConversionFlag { /// No conversion @@ -314,32 +305,57 @@ pub enum ConversionFlag { /// Converts by calling `repr()`. Repr = b'r', } -enum_oparg!(ConversionFlag); + +impl OpArgType for ConversionFlag { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match u8::try_from(x).ok()? { + 0 => Self::None, + b's' => Self::Str, + b'a' => Self::Ascii, + b'r' => Self::Repr, + _ => return None, + }) + } +} impl TryFrom for ConversionFlag { type Error = usize; fn try_from(b: usize) -> Result { - u8::try_from(b) - .ok() - .and_then(|b| Self::try_from(b).ok()) - .ok_or(b) + u32::try_from(b).ok().and_then(Self::from_oparg).ok_or(b) } } /// The kind of Raise that occurred. -#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[repr(u8)] pub enum RaiseKind { Reraise, Raise, RaiseCause, } -enum_oparg!(RaiseKind); + +impl OpArgType for RaiseKind { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match x { + 0 => Self::Reraise, + 1 => Self::Raise, + 2 => Self::RaiseCause, + _ => return None, + }) + } +} pub type NameIdx = u32; /// A Single bytecode instruction. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u8)] pub enum Instruction { /// Importing by name ImportName { @@ -562,7 +578,29 @@ pub enum Instruction { } const _: () = assert!(mem::size_of::() == 1); -#[derive(Copy, Clone, Serialize, Deserialize)] +impl From for u8 { + #[inline] + fn from(ins: Instruction) -> u8 { + // SAFETY: there's no padding bits + unsafe { std::mem::transmute::(ins) } + } +} + +impl TryFrom for Instruction { + type Error = crate::marshal::MarshalError; + + #[inline] + fn try_from(value: u8) -> Result { + if value <= u8::from(Instruction::ExtendedArg) { + Ok(unsafe { std::mem::transmute::(value) }) + } else { + Err(crate::marshal::MarshalError::InvalidBytecode) + } + } +} + +#[derive(Copy, Clone)] +#[repr(C)] pub struct CodeUnit { pub op: Instruction, pub arg: OpArgByte, @@ -579,7 +617,6 @@ impl CodeUnit { use self::Instruction::*; bitflags! { - #[derive(Serialize, Deserialize)] pub struct MakeFunctionFlags: u8 { const CLOSURE = 0x01; const ANNOTATIONS = 0x02; @@ -607,7 +644,7 @@ impl OpArgType for MakeFunctionFlags { /// let b = ConstantData::Boolean {value: false}; /// assert_ne!(a, b); /// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] pub enum ConstantData { Tuple { elements: Vec }, Integer { value: BigInt }, @@ -677,36 +714,40 @@ pub enum BorrowedConstant<'a, C: Constant> { Str { value: &'a str }, Bytes { value: &'a [u8] }, Code { code: &'a CodeObject }, - Tuple { elements: BorrowedTupleIter<'a, C> }, + Tuple { elements: &'a [C] }, None, Ellipsis, } -type BorrowedTupleIter<'a, C> = Box> + 'a>; +impl Copy for BorrowedConstant<'_, C> {} +impl Clone for BorrowedConstant<'_, C> { + fn clone(&self) -> Self { + *self + } +} impl BorrowedConstant<'_, C> { - // takes `self` because we need to consume the iterator - pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result { + pub fn fmt_display(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { BorrowedConstant::Integer { value } => write!(f, "{value}"), BorrowedConstant::Float { value } => write!(f, "{value}"), BorrowedConstant::Complex { value } => write!(f, "{value}"), BorrowedConstant::Boolean { value } => { - write!(f, "{}", if value { "True" } else { "False" }) + write!(f, "{}", if *value { "True" } else { "False" }) } BorrowedConstant::Str { value } => write!(f, "{value:?}"), - BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()), + BorrowedConstant::Bytes { value } => write!(f, "b\"{}\"", value.escape_ascii()), BorrowedConstant::Code { code } => write!(f, "{code:?}"), BorrowedConstant::Tuple { elements } => { write!(f, "(")?; let mut first = true; - for c in elements { + for c in *elements { if first { first = false } else { write!(f, ", ")?; } - c.fmt_display(f)?; + c.borrow_constant().fmt_display(f)?; } write!(f, ")") } @@ -733,7 +774,10 @@ impl BorrowedConstant<'_, C> { code: Box::new(code.map_clone_bag(&BasicBag)), }, BorrowedConstant::Tuple { elements } => Tuple { - elements: elements.map(BorrowedConstant::to_owned).collect(), + elements: elements + .iter() + .map(|c| c.borrow_constant().to_owned()) + .collect(), }, BorrowedConstant::None => None, BorrowedConstant::Ellipsis => Ellipsis, @@ -742,7 +786,7 @@ impl BorrowedConstant<'_, C> { } /// The possible comparison operators -#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum ComparisonOperator { // be intentional with bits so that we can do eval_ord with just a bitwise and @@ -754,9 +798,25 @@ pub enum ComparisonOperator { LessOrEqual = 0b101, GreaterOrEqual = 0b110, } -enum_oparg!(ComparisonOperator); -#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +impl OpArgType for ComparisonOperator { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match x { + 0b001 => Self::Less, + 0b010 => Self::Greater, + 0b011 => Self::NotEqual, + 0b100 => Self::Equal, + 0b101 => Self::LessOrEqual, + 0b110 => Self::GreaterOrEqual, + _ => return None, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum TestOperator { In, @@ -766,7 +826,22 @@ pub enum TestOperator { /// two exceptions that match? ExceptionMatch, } -enum_oparg!(TestOperator); + +impl OpArgType for TestOperator { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match x { + 0 => Self::In, + 1 => Self::NotIn, + 2 => Self::Is, + 3 => Self::IsNot, + 4 => Self::ExceptionMatch, + _ => return None, + }) + } +} /// The possible Binary operators /// # Examples @@ -776,7 +851,7 @@ enum_oparg!(TestOperator); /// use rustpython_compiler_core::BinaryOperator::Add; /// let op = BinaryOperation {op: Add}; /// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum BinaryOperator { Power, @@ -793,10 +868,33 @@ pub enum BinaryOperator { Xor, Or, } -enum_oparg!(BinaryOperator); + +impl OpArgType for BinaryOperator { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match x { + 0 => Self::Power, + 1 => Self::Multiply, + 2 => Self::MatrixMultiply, + 3 => Self::Divide, + 4 => Self::FloorDivide, + 5 => Self::Modulo, + 6 => Self::Add, + 7 => Self::Subtract, + 8 => Self::Lshift, + 9 => Self::Rshift, + 10 => Self::And, + 11 => Self::Xor, + 12 => Self::Or, + _ => return None, + }) + } +} /// The possible unary operators -#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum UnaryOperator { Not, @@ -804,7 +902,21 @@ pub enum UnaryOperator { Minus, Plus, } -enum_oparg!(UnaryOperator); + +impl OpArgType for UnaryOperator { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match x { + 0 => Self::Not, + 1 => Self::Invert, + 2 => Self::Minus, + 3 => Self::Plus, + _ => return None, + }) + } +} #[derive(Copy, Clone)] pub struct UnpackExArgs { @@ -867,10 +979,10 @@ impl CodeObject { /// Get all arguments of the code object /// like inspect.getargs pub fn arg_names(&self) -> Arguments { - let nargs = self.arg_count; - let nkwargs = self.kwonlyarg_count; + let nargs = self.arg_count as usize; + let nkwargs = self.kwonlyarg_count as usize; let mut varargspos = nargs + nkwargs; - let posonlyargs = &self.varnames[..self.posonlyarg_count]; + let posonlyargs = &self.varnames[..self.posonlyarg_count as usize]; let args = &self.varnames[..nargs]; let kwonlyargs = &self.varnames[nargs..varargspos]; @@ -1033,49 +1145,23 @@ impl CodeObject { } } -/// Error that occurs during code deserialization -#[derive(Debug)] -#[non_exhaustive] -pub enum CodeDeserializeError { - /// Unexpected End Of File - Eof, - /// Invalid Bytecode - Other, -} - -impl fmt::Display for CodeDeserializeError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::Eof => f.write_str("unexpected end of data"), - Self::Other => f.write_str("invalid bytecode"), - } - } -} - -impl std::error::Error for CodeDeserializeError {} - impl CodeObject { /// Load a code object from bytes - pub fn from_bytes(data: &[u8]) -> Result { + pub fn from_bytes(data: &[u8]) -> Result { use lz4_flex::block::DecompressError; let raw_bincode = lz4_flex::decompress_size_prepended(data).map_err(|e| match e { DecompressError::OutputTooSmall { .. } | DecompressError::ExpectedAnotherByte => { - CodeDeserializeError::Eof + MarshalError::Eof } - _ => CodeDeserializeError::Other, + _ => MarshalError::InvalidBytecode, })?; - let data = bincode::deserialize(&raw_bincode).map_err(|e| match *e { - bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - CodeDeserializeError::Eof - } - _ => CodeDeserializeError::Other, - })?; - Ok(data) + marshal::deserialize_code(&mut &raw_bincode[..], BasicBag) } /// Serialize this bytecode to bytes. pub fn to_bytes(&self) -> Vec { - let data = bincode::serialize(&self).expect("CodeObject is not serializable"); + let mut data = Vec::new(); + marshal::serialize_code(&mut data, self); lz4_flex::compress_prepend_size(&data) } } @@ -1466,7 +1552,7 @@ impl fmt::Debug for CodeObject { } /// A frozen module. Holds a code object and whether it is part of a package -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug)] pub struct FrozenModule { pub code: CodeObject, pub package: bool, @@ -1474,34 +1560,28 @@ pub struct FrozenModule { pub mod frozen_lib { use super::*; - use bincode::{options, Options}; - use std::io; + use marshal::{Read, Write}; /// Decode a library to a iterable of frozen modules pub fn decode_lib(bytes: &[u8]) -> FrozenModulesIter { let data = lz4_flex::decompress_size_prepended(bytes).unwrap(); - let r = VecReader { data, pos: 0 }; - let mut de = bincode::Deserializer::with_bincode_read(r, options()); - let len = u64::deserialize(&mut de).unwrap().try_into().unwrap(); - FrozenModulesIter { len, de } + let mut data = marshal::Cursor { data, position: 0 }; + let remaining = data.read_u32().unwrap(); + FrozenModulesIter { remaining, data } } pub struct FrozenModulesIter { - len: usize, - // ideally this could be a SeqAccess, but I think that would require existential types - de: bincode::Deserializer, + remaining: u32, + data: marshal::Cursor>, } impl Iterator for FrozenModulesIter { type Item = (String, FrozenModule); fn next(&mut self) -> Option { - // manually mimic bincode's seq encoding, which is - // This probably won't change (bincode doesn't require padding or anything), but - // it's not guaranteed by semver as far as I can tell - if self.len > 0 { - let entry = Deserialize::deserialize(&mut self.de).unwrap(); - self.len -= 1; + if self.remaining > 0 { + let entry = read_entry(&mut self.data).unwrap(); + self.remaining -= 1; Some(entry) } else { None @@ -1509,12 +1589,19 @@ pub mod frozen_lib { } fn size_hint(&self) -> (usize, Option) { - (self.len, Some(self.len)) + (self.remaining as usize, Some(self.remaining as usize)) } } - impl ExactSizeIterator for FrozenModulesIter {} + fn read_entry(rdr: &mut impl Read) -> Result<(String, FrozenModule), marshal::MarshalError> { + let len = rdr.read_u32()?; + let name = rdr.read_str(len)?.to_owned(); + let code = marshal::deserialize_code(rdr, BasicBag)?; + let package = rdr.read_u8()? != 0; + Ok((name, FrozenModule { code, package })) + } + /// Encode the given iterator of frozen modules into a compressed vector of bytes pub fn encode_lib<'a, I>(lib: I) -> Vec where @@ -1522,82 +1609,25 @@ pub mod frozen_lib { I::IntoIter: ExactSizeIterator + Clone, { let iter = lib.into_iter(); - let data = options().serialize(&SerializeLib { iter }).unwrap(); + let mut data = Vec::new(); + write_lib(&mut data, iter); lz4_flex::compress_prepend_size(&data) } - struct SerializeLib { - iter: I, - } - - impl<'a, I> Serialize for SerializeLib - where - I: ExactSizeIterator + Clone, - { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.collect_seq(self.iter.clone()) + fn write_lib<'a>( + buf: &mut impl Write, + lib: impl ExactSizeIterator, + ) { + marshal::write_len(buf, lib.len()); + for (name, module) in lib { + write_entry(buf, name, module); } } - /// Owned version of bincode::de::read::SliceReader<'a> - struct VecReader { - data: Vec, - pos: usize, - } - - impl io::Read for VecReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let mut subslice = &self.data[self.pos..]; - let n = io::Read::read(&mut subslice, buf)?; - self.pos += n; - Ok(n) - } - fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { - self.get_byte_slice(buf.len()) - .map(|data| buf.copy_from_slice(data)) - } - } - - impl VecReader { - #[inline(always)] - fn get_byte_slice(&mut self, length: usize) -> io::Result<&[u8]> { - let subslice = &self.data[self.pos..]; - match subslice.get(..length) { - Some(ret) => { - self.pos += length; - Ok(ret) - } - None => Err(io::ErrorKind::UnexpectedEof.into()), - } - } - } - - impl<'storage> bincode::BincodeRead<'storage> for VecReader { - fn forward_read_str(&mut self, length: usize, visitor: V) -> bincode::Result - where - V: serde::de::Visitor<'storage>, - { - let bytes = self.get_byte_slice(length)?; - match ::std::str::from_utf8(bytes) { - Ok(s) => visitor.visit_str(s), - Err(e) => Err(bincode::ErrorKind::InvalidUtf8Encoding(e).into()), - } - } - - fn get_byte_buffer(&mut self, length: usize) -> bincode::Result> { - self.get_byte_slice(length) - .map(|x| x.to_vec()) - .map_err(Into::into) - } - - fn forward_read_bytes(&mut self, length: usize, visitor: V) -> bincode::Result - where - V: serde::de::Visitor<'storage>, - { - visitor.visit_bytes(self.get_byte_slice(length)?) - } + fn write_entry(buf: &mut impl Write, name: &str, module: &FrozenModule) { + marshal::write_len(buf, name.len()); + buf.write_slice(name.as_bytes()); + marshal::serialize_code(buf, &module.code); + buf.write_u8(module.package as u8); } } diff --git a/compiler/core/src/error.rs b/compiler/core/src/error.rs index 9619e1d8c..813ca9832 100644 --- a/compiler/core/src/error.rs +++ b/compiler/core/src/error.rs @@ -1,4 +1,5 @@ use crate::Location; +use std::error::Error as StdError; use std::fmt::Display; #[derive(Debug, PartialEq, Eq)] @@ -16,7 +17,14 @@ impl std::ops::Deref for BaseError { } } -impl std::error::Error for BaseError where T: std::fmt::Display + std::fmt::Debug {} +impl StdError for BaseError +where + T: StdError + 'static, +{ + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(&self.error) + } +} impl Display for BaseError where @@ -51,12 +59,18 @@ impl BaseError { } } -#[derive(Debug, thiserror::Error)] +#[derive(Debug)] pub struct CompileError { pub body: BaseError, pub statement: Option, } +impl StdError for CompileError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.body.source() + } +} + impl std::ops::Deref for CompileError { type Target = BaseError; fn deref(&self) -> &Self::Target { diff --git a/compiler/core/src/lib.rs b/compiler/core/src/lib.rs index e1f0a5b35..c431095f4 100644 --- a/compiler/core/src/lib.rs +++ b/compiler/core/src/lib.rs @@ -4,6 +4,7 @@ mod bytecode; mod error; mod location; +pub mod marshal; mod mode; pub use bytecode::*; diff --git a/compiler/core/src/location.rs b/compiler/core/src/location.rs index d9cd59bea..2a2d029d3 100644 --- a/compiler/core/src/location.rs +++ b/compiler/core/src/location.rs @@ -1,7 +1,5 @@ -use serde::{Deserialize, Serialize}; - /// Sourcecode location. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct Location { pub(super) row: u32, pub(super) column: u32, diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs new file mode 100644 index 000000000..69d887934 --- /dev/null +++ b/compiler/core/src/marshal.rs @@ -0,0 +1,615 @@ +use core::fmt; +use std::convert::Infallible; + +use num_bigint::{BigInt, Sign}; +use num_complex::Complex64; + +use crate::{bytecode::*, Location}; + +pub const FORMAT_VERSION: u32 = 4; + +#[derive(Debug)] +pub enum MarshalError { + /// Unexpected End Of File + Eof, + /// Invalid Bytecode + InvalidBytecode, + /// Invalid utf8 in string + InvalidUtf8, + /// Bad type marker + BadType, +} + +impl fmt::Display for MarshalError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Eof => f.write_str("unexpected end of data"), + Self::InvalidBytecode => f.write_str("invalid bytecode"), + Self::InvalidUtf8 => f.write_str("invalid utf8"), + Self::BadType => f.write_str("bad type marker"), + } + } +} + +impl From for MarshalError { + fn from(_: std::str::Utf8Error) -> Self { + Self::InvalidUtf8 + } +} + +impl std::error::Error for MarshalError {} + +type Result = std::result::Result; + +#[repr(u8)] +enum Type { + // Null = b'0', + None = b'N', + False = b'F', + True = b'T', + StopIter = b'S', + Ellipsis = b'.', + Int = b'i', + Float = b'g', + Complex = b'y', + // Long = b'l', // i32 + Bytes = b's', // = TYPE_STRING + // Interned = b't', + // Ref = b'r', + Tuple = b'(', + List = b'[', + Dict = b'{', + Code = b'c', + Unicode = b'u', + // Unknown = b'?', + Set = b'<', + FrozenSet = b'>', + Ascii = b'a', + // AsciiInterned = b'A', + // SmallTuple = b')', + // ShortAscii = b'z', + // ShortAsciiInterned = b'Z', +} +// const FLAG_REF: u8 = b'\x80'; + +impl TryFrom for Type { + type Error = MarshalError; + fn try_from(value: u8) -> Result { + use Type::*; + Ok(match value { + // b'0' => Null, + b'N' => None, + b'F' => False, + b'T' => True, + b'S' => StopIter, + b'.' => Ellipsis, + b'i' => Int, + b'g' => Float, + b'y' => Complex, + // b'l' => Long, + b's' => Bytes, + // b't' => Interned, + // b'r' => Ref, + b'(' => Tuple, + b'[' => List, + b'{' => Dict, + b'c' => Code, + b'u' => Unicode, + // b'?' => Unknown, + b'<' => Set, + b'>' => FrozenSet, + b'a' => Ascii, + // b'A' => AsciiInterned, + // b')' => SmallTuple, + // b'z' => ShortAscii, + // b'Z' => ShortAsciiInterned, + _ => return Err(MarshalError::BadType), + }) + } +} + +pub trait Read { + fn read_slice(&mut self, n: u32) -> Result<&[u8]>; + fn read_array(&mut self) -> Result<&[u8; N]> { + self.read_slice(N as u32).map(|s| s.try_into().unwrap()) + } + fn read_str(&mut self, len: u32) -> Result<&str> { + Ok(std::str::from_utf8(self.read_slice(len)?)?) + } + fn read_u8(&mut self) -> Result { + Ok(u8::from_le_bytes(*self.read_array()?)) + } + fn read_u16(&mut self) -> Result { + Ok(u16::from_le_bytes(*self.read_array()?)) + } + fn read_u32(&mut self) -> Result { + Ok(u32::from_le_bytes(*self.read_array()?)) + } + fn read_u64(&mut self) -> Result { + Ok(u64::from_le_bytes(*self.read_array()?)) + } +} + +impl Read for &[u8] { + fn read_slice(&mut self, n: u32) -> Result<&[u8]> { + let data = self.get(..n as usize).ok_or(MarshalError::Eof)?; + *self = &self[n as usize..]; + Ok(data) + } +} + +pub struct Cursor { + pub data: B, + pub position: usize, +} + +impl> Read for Cursor { + fn read_slice(&mut self, n: u32) -> Result<&[u8]> { + let data = &self.data.as_ref()[self.position..]; + let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?; + self.position += n as usize; + Ok(slice) + } +} + +pub fn deserialize_code( + rdr: &mut R, + bag: Bag, +) -> Result> { + let len = rdr.read_u32()?; + let instructions = rdr.read_slice(len * 2)?; + let instructions = instructions + .chunks_exact(2) + .map(|cu| { + let op = Instruction::try_from(cu[0])?; + let arg = OpArgByte(cu[1]); + Ok(CodeUnit { op, arg }) + }) + .collect::>>()?; + + let len = rdr.read_u32()?; + let locations = (0..len) + .map(|_| { + Ok(Location { + row: rdr.read_u32()?, + column: rdr.read_u32()?, + }) + }) + .collect::>>()?; + + let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?); + + let posonlyarg_count = rdr.read_u32()?; + let arg_count = rdr.read_u32()?; + let kwonlyarg_count = rdr.read_u32()?; + + let len = rdr.read_u32()?; + let source_path = bag.make_name(rdr.read_str(len)?); + + let first_line_number = rdr.read_u32()?; + let max_stackdepth = rdr.read_u32()?; + + let len = rdr.read_u32()?; + let obj_name = bag.make_name(rdr.read_str(len)?); + + let len = rdr.read_u32()?; + let cell2arg = (len != 0) + .then(|| { + (0..len) + .map(|_| Ok(rdr.read_u32()? as i32)) + .collect::>>() + }) + .transpose()?; + + let len = rdr.read_u32()?; + let constants = (0..len) + .map(|_| deserialize_value(rdr, bag)) + .collect::>>()?; + + let mut read_names = || { + let len = rdr.read_u32()?; + (0..len) + .map(|_| { + let len = rdr.read_u32()?; + Ok(bag.make_name(rdr.read_str(len)?)) + }) + .collect::>>() + }; + + let names = read_names()?; + let varnames = read_names()?; + let cellvars = read_names()?; + let freevars = read_names()?; + + Ok(CodeObject { + instructions, + locations, + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path, + first_line_number, + max_stackdepth, + obj_name, + cell2arg, + constants, + names, + varnames, + cellvars, + freevars, + }) +} + +pub trait MarshalBag: Copy { + type Value; + fn make_bool(&self, value: bool) -> Self::Value; + fn make_none(&self) -> Self::Value; + fn make_ellipsis(&self) -> Self::Value; + fn make_float(&self, value: f64) -> Self::Value; + fn make_complex(&self, value: Complex64) -> Self::Value; + fn make_str(&self, value: &str) -> Self::Value; + fn make_bytes(&self, value: &[u8]) -> Self::Value; + fn make_int(&self, value: BigInt) -> Self::Value; + fn make_tuple(&self, elements: impl Iterator) -> Self::Value; + fn make_code( + &self, + code: CodeObject<::Constant>, + ) -> Self::Value; + fn make_stop_iter(&self) -> Result; + fn make_list(&self, it: impl Iterator) -> Result; + fn make_set(&self, it: impl Iterator) -> Result; + fn make_frozenset(&self, it: impl Iterator) -> Result; + fn make_dict( + &self, + it: impl Iterator, + ) -> Result; + type ConstantBag: ConstantBag; + fn constant_bag(self) -> Self::ConstantBag; +} + +impl MarshalBag for Bag { + type Value = Bag::Constant; + fn make_bool(&self, value: bool) -> Self::Value { + self.make_constant::(BorrowedConstant::Boolean { value }) + } + fn make_none(&self) -> Self::Value { + self.make_constant::(BorrowedConstant::None) + } + fn make_ellipsis(&self) -> Self::Value { + self.make_constant::(BorrowedConstant::Ellipsis) + } + fn make_float(&self, value: f64) -> Self::Value { + self.make_constant::(BorrowedConstant::Float { value }) + } + fn make_complex(&self, value: Complex64) -> Self::Value { + self.make_constant::(BorrowedConstant::Complex { value }) + } + fn make_str(&self, value: &str) -> Self::Value { + self.make_constant::(BorrowedConstant::Str { value }) + } + fn make_bytes(&self, value: &[u8]) -> Self::Value { + self.make_constant::(BorrowedConstant::Bytes { value }) + } + fn make_int(&self, value: BigInt) -> Self::Value { + self.make_int(value) + } + fn make_tuple(&self, elements: impl Iterator) -> Self::Value { + self.make_tuple(elements) + } + fn make_code( + &self, + code: CodeObject<::Constant>, + ) -> Self::Value { + self.make_code(code) + } + fn make_stop_iter(&self) -> Result { + Err(MarshalError::BadType) + } + fn make_list(&self, _: impl Iterator) -> Result { + Err(MarshalError::BadType) + } + fn make_set(&self, _: impl Iterator) -> Result { + Err(MarshalError::BadType) + } + fn make_frozenset(&self, _: impl Iterator) -> Result { + Err(MarshalError::BadType) + } + fn make_dict( + &self, + _: impl Iterator, + ) -> Result { + Err(MarshalError::BadType) + } + type ConstantBag = Self; + fn constant_bag(self) -> Self::ConstantBag { + self + } +} + +pub fn deserialize_value(rdr: &mut R, bag: Bag) -> Result { + let typ = Type::try_from(rdr.read_u8()?)?; + let value = match typ { + Type::True => bag.make_bool(true), + Type::False => bag.make_bool(false), + Type::None => bag.make_none(), + Type::StopIter => bag.make_stop_iter()?, + Type::Ellipsis => bag.make_ellipsis(), + Type::Int => { + let len = rdr.read_u32()? as i32; + let sign = if len < 0 { Sign::Minus } else { Sign::Plus }; + let bytes = rdr.read_slice(len.unsigned_abs())?; + let int = BigInt::from_bytes_le(sign, bytes); + bag.make_int(int) + } + Type::Float => { + let value = f64::from_bits(rdr.read_u64()?); + bag.make_float(value) + } + Type::Complex => { + let re = f64::from_bits(rdr.read_u64()?); + let im = f64::from_bits(rdr.read_u64()?); + let value = Complex64 { re, im }; + bag.make_complex(value) + } + Type::Ascii | Type::Unicode => { + let len = rdr.read_u32()?; + let value = rdr.read_str(len)?; + bag.make_str(value) + } + Type::Tuple => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } + Type::List => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_list(it))?? + } + Type::Set => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_set(it))?? + } + Type::FrozenSet => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_frozenset(it))?? + } + Type::Dict => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| { + let k = deserialize_value(rdr, bag)?; + let v = deserialize_value(rdr, bag)?; + Ok::<_, MarshalError>((k, v)) + }); + itertools::process_results(it, |it| bag.make_dict(it))?? + } + Type::Bytes => { + // Following CPython, after marshaling, byte arrays are converted into bytes. + let len = rdr.read_u32()?; + let value = rdr.read_slice(len)?; + bag.make_bytes(value) + } + Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?), + }; + Ok(value) +} + +pub trait Dumpable: Sized { + type Error; + type Constant: Constant; + fn with_dump(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result; +} + +pub enum DumpableValue<'a, D: Dumpable> { + Integer(&'a BigInt), + Float(f64), + Complex(Complex64), + Boolean(bool), + Str(&'a str), + Bytes(&'a [u8]), + Code(&'a CodeObject), + Tuple(&'a [D]), + None, + Ellipsis, + StopIter, + List(&'a [D]), + Set(&'a [D]), + Frozenset(&'a [D]), + Dict(&'a [(D, D)]), +} + +impl<'a, C: Constant> From> for DumpableValue<'a, C> { + fn from(c: BorrowedConstant<'a, C>) -> Self { + match c { + BorrowedConstant::Integer { value } => Self::Integer(value), + BorrowedConstant::Float { value } => Self::Float(value), + BorrowedConstant::Complex { value } => Self::Complex(value), + BorrowedConstant::Boolean { value } => Self::Boolean(value), + BorrowedConstant::Str { value } => Self::Str(value), + BorrowedConstant::Bytes { value } => Self::Bytes(value), + BorrowedConstant::Code { code } => Self::Code(code), + BorrowedConstant::Tuple { elements } => Self::Tuple(elements), + BorrowedConstant::None => Self::None, + BorrowedConstant::Ellipsis => Self::Ellipsis, + } + } +} + +impl Dumpable for C { + type Error = Infallible; + type Constant = Self; + #[inline(always)] + fn with_dump(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result { + Ok(f(self.borrow_constant().into())) + } +} + +pub trait Write { + fn write_slice(&mut self, slice: &[u8]); + fn write_u8(&mut self, v: u8) { + self.write_slice(&v.to_le_bytes()) + } + fn write_u16(&mut self, v: u16) { + self.write_slice(&v.to_le_bytes()) + } + fn write_u32(&mut self, v: u32) { + self.write_slice(&v.to_le_bytes()) + } + fn write_u64(&mut self, v: u64) { + self.write_slice(&v.to_le_bytes()) + } +} + +impl Write for Vec { + fn write_slice(&mut self, slice: &[u8]) { + self.extend_from_slice(slice) + } +} + +pub(crate) fn write_len(buf: &mut W, len: usize) { + let Ok(len) = len.try_into() else { panic!("too long to serialize") }; + buf.write_u32(len); +} + +pub fn serialize_value( + buf: &mut W, + constant: DumpableValue<'_, D>, +) -> Result<(), D::Error> { + match constant { + DumpableValue::Integer(int) => { + buf.write_u8(Type::Int as u8); + let (sign, bytes) = int.to_bytes_le(); + let len: i32 = bytes.len().try_into().expect("too long to serialize"); + let len = if sign == Sign::Minus { -len } else { len }; + buf.write_u32(len as u32); + buf.write_slice(&bytes); + } + DumpableValue::Float(f) => { + buf.write_u8(Type::Float as u8); + buf.write_u64(f.to_bits()); + } + DumpableValue::Complex(c) => { + buf.write_u8(Type::Complex as u8); + buf.write_u64(c.re.to_bits()); + buf.write_u64(c.im.to_bits()); + } + DumpableValue::Boolean(b) => { + buf.write_u8(if b { Type::True } else { Type::False } as u8); + } + DumpableValue::Str(s) => { + buf.write_u8(Type::Unicode as u8); + write_len(buf, s.len()); + buf.write_slice(s.as_bytes()); + } + DumpableValue::Bytes(b) => { + buf.write_u8(Type::Bytes as u8); + write_len(buf, b.len()); + buf.write_slice(b); + } + DumpableValue::Code(c) => { + buf.write_u8(Type::Code as u8); + serialize_code(buf, c); + } + DumpableValue::Tuple(tup) => { + buf.write_u8(Type::Tuple as u8); + write_len(buf, tup.len()); + for val in tup { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::None => { + buf.write_u8(Type::None as u8); + } + DumpableValue::Ellipsis => { + buf.write_u8(Type::Ellipsis as u8); + } + DumpableValue::StopIter => { + buf.write_u8(Type::StopIter as u8); + } + DumpableValue::List(l) => { + buf.write_u8(Type::List as u8); + write_len(buf, l.len()); + for val in l { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Set(set) => { + buf.write_u8(Type::Set as u8); + write_len(buf, set.len()); + for val in set { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Frozenset(set) => { + buf.write_u8(Type::FrozenSet as u8); + write_len(buf, set.len()); + for val in set { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Dict(d) => { + buf.write_u8(Type::Dict as u8); + write_len(buf, d.len()); + for (k, v) in d { + k.with_dump(|val| serialize_value(buf, val))??; + v.with_dump(|val| serialize_value(buf, val))??; + } + } + } + Ok(()) +} + +pub fn serialize_code(buf: &mut W, code: &CodeObject) { + write_len(buf, code.instructions.len()); + // SAFETY: it's ok to transmute CodeUnit to [u8; 2] + let (_, instructions_bytes, _) = unsafe { code.instructions.align_to() }; + buf.write_slice(instructions_bytes); + + write_len(buf, code.locations.len()); + for loc in &*code.locations { + buf.write_u32(loc.row); + buf.write_u32(loc.column); + } + + buf.write_u16(code.flags.bits()); + + buf.write_u32(code.posonlyarg_count); + buf.write_u32(code.arg_count); + buf.write_u32(code.kwonlyarg_count); + + write_len(buf, code.source_path.as_ref().len()); + buf.write_slice(code.source_path.as_ref().as_bytes()); + + buf.write_u32(code.first_line_number); + buf.write_u32(code.max_stackdepth); + + write_len(buf, code.obj_name.as_ref().len()); + buf.write_slice(code.obj_name.as_ref().as_bytes()); + + let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]); + write_len(buf, cell2arg.len()); + for &i in cell2arg { + buf.write_u32(i as u32) + } + + write_len(buf, code.constants.len()); + for constant in &*code.constants { + serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}) + } + + let mut write_names = |names: &[C::Name]| { + write_len(buf, names.len()); + for name in names { + write_len(buf, name.as_ref().len()); + buf.write_slice(name.as_ref().as_bytes()); + } + }; + + write_names(&code.names); + write_names(&code.varnames); + write_names(&code.cellvars); + write_names(&code.freevars); +} diff --git a/compiler/parser/Cargo.toml b/compiler/parser/Cargo.toml index 40bf75e50..026616d39 100644 --- a/compiler/parser/Cargo.toml +++ b/compiler/parser/Cargo.toml @@ -26,7 +26,6 @@ itertools = { workspace = true } log = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } -thiserror = { workspace = true } unicode_names2 = { workspace = true } unic-emoji-char = "0.9.0" diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs index c2c4a3831..04875c44e 100644 --- a/compiler/parser/src/parser.rs +++ b/compiler/parser/src/parser.rs @@ -200,7 +200,7 @@ pub fn parse_tokens( pub type ParseError = rustpython_compiler_core::BaseError; /// Represents the different types of errors that can occur during parsing. -#[derive(Debug, PartialEq, thiserror::Error)] +#[derive(Debug, PartialEq)] pub enum ParseErrorType { /// Parser encountered an unexpected end of input Eof, @@ -215,6 +215,8 @@ pub enum ParseErrorType { Lexical(LexicalErrorType), } +impl std::error::Error for ParseErrorType {} + // Convert `lalrpop_util::ParseError` to our internal type fn parse_error_from_lalrpop( err: LalrpopError, diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 804ce10ec..27c158727 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -7,12 +7,40 @@ use rustpython_parser::{ pub use rustpython_codegen::compile::CompileOpts; pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode}; -#[derive(Debug, thiserror::Error)] +use std::error::Error as StdError; +use std::fmt; + +#[derive(Debug)] pub enum CompileErrorType { - #[error(transparent)] - Codegen(#[from] rustpython_codegen::error::CodegenErrorType), - #[error(transparent)] - Parse(#[from] parser::ParseErrorType), + Codegen(rustpython_codegen::error::CodegenErrorType), + Parse(parser::ParseErrorType), +} + +impl StdError for CompileErrorType { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + match self { + CompileErrorType::Codegen(e) => e.source(), + CompileErrorType::Parse(e) => e.source(), + } + } +} +impl fmt::Display for CompileErrorType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + CompileErrorType::Codegen(e) => e.fmt(f), + CompileErrorType::Parse(e) => e.fmt(f), + } + } +} +impl From for CompileErrorType { + fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self { + CompileErrorType::Codegen(source) + } +} +impl From for CompileErrorType { + fn from(source: parser::ParseErrorType) -> Self { + CompileErrorType::Parse(source) + } } pub type CompileError = rustpython_compiler_core::CompileError; diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 841b5c519..48d43e873 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -21,6 +21,7 @@ compiler = ["parser", "codegen", "rustpython-compiler"] ast = ["rustpython-ast"] codegen = ["rustpython-codegen", "ast"] parser = ["rustpython-parser", "ast"] +serde = ["dep:serde"] [dependencies] rustpython-compiler = { path = "../compiler", optional = true, version = "0.2.0" } @@ -47,8 +48,8 @@ itertools = { workspace = true } libc = { workspace = true } log = { workspace = true } nix = { workspace = true } -num-bigint = { workspace = true, features = ["serde"] } -num-complex = { workspace = true, features = ["serde"] } +num-bigint = { workspace = true } +num-complex = { workspace = true } num-integer = { workspace = true } num-rational = { workspace = true } num-traits = { workspace = true } @@ -57,7 +58,7 @@ once_cell = { workspace = true } parking_lot = { workspace = true } paste = { workspace = true } rand = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, optional = true } static_assertions = { workspace = true } thiserror = { workspace = true } thread_local = { workspace = true } diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 2258af473..885e57b4f 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -17,15 +17,15 @@ use std::{borrow::Borrow, fmt, ops::Deref}; #[derive(FromArgs)] pub struct ReplaceArgs { #[pyarg(named, optional)] - co_posonlyargcount: OptionalArg, + co_posonlyargcount: OptionalArg, #[pyarg(named, optional)] - co_argcount: OptionalArg, + co_argcount: OptionalArg, #[pyarg(named, optional)] - co_kwonlyargcount: OptionalArg, + co_kwonlyargcount: OptionalArg, #[pyarg(named, optional)] co_filename: OptionalArg, #[pyarg(named, optional)] - co_firstlineno: OptionalArg, + co_firstlineno: OptionalArg, #[pyarg(named, optional)] co_consts: OptionalArg>, #[pyarg(named, optional)] @@ -39,6 +39,7 @@ pub struct ReplaceArgs { } #[derive(Clone)] +#[repr(transparent)] pub struct Literal(PyObjectRef); impl Borrow for Literal { @@ -77,9 +78,9 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant { BorrowedConstant::Code { code: &c.code } } ref t @ super::tuple::PyTuple => { - BorrowedConstant::Tuple { - elements: Box::new(t.iter().map(|o| borrow_obj_constant(o))), - } + let elements = t.as_slice(); + let elements = unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) }; + BorrowedConstant::Tuple { elements } } super::singletons::PyNone => BorrowedConstant::None, super::slice::PyEllipsis => BorrowedConstant::Ellipsis, @@ -117,8 +118,8 @@ impl ConstantBag for PyObjBag<'_> { } bytecode::BorrowedConstant::Tuple { elements } => { let elements = elements - .into_iter() - .map(|constant| self.make_constant(constant).0) + .iter() + .map(|constant| self.make_constant(constant.borrow_constant()).0) .collect(); ctx.new_tuple(elements).into() } @@ -131,6 +132,18 @@ impl ConstantBag for PyObjBag<'_> { fn make_name(&self, name: &str) -> &'static PyStrInterned { self.0.intern_str(name) } + + fn make_int(&self, value: num_bigint::BigInt) -> Self::Constant { + Literal(self.0.new_int(value).into()) + } + + fn make_tuple(&self, elements: impl Iterator) -> Self::Constant { + Literal(self.0.new_tuple(elements.map(|lit| lit.0).collect()).into()) + } + + fn make_code(&self, code: CodeObject) -> Self::Constant { + Literal(self.0.new_code(code).into()) + } } pub type CodeObject = bytecode::CodeObject; @@ -205,12 +218,12 @@ impl PyRef { #[pygetset] fn co_posonlyargcount(self) -> usize { - self.code.posonlyarg_count + self.code.posonlyarg_count as usize } #[pygetset] fn co_argcount(self) -> usize { - self.code.arg_count + self.code.arg_count as usize } #[pygetset] @@ -237,12 +250,12 @@ impl PyRef { #[pygetset] fn co_firstlineno(self) -> usize { - self.code.first_line_number + self.code.first_line_number as usize } #[pygetset] fn co_kwonlyargcount(self) -> usize { - self.code.kwonlyarg_count + self.code.kwonlyarg_count as usize } #[pygetset] diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index 3f86b48d1..3aaa4d697 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -28,6 +28,12 @@ pub struct PyComplex { value: Complex64, } +impl PyComplex { + pub fn to_complex64(self) -> Complex64 { + self.value + } +} + impl PyPayload for PyComplex { fn class(vm: &VirtualMachine) -> &'static Py { vm.ctx.types.complex_type diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index 7fd8cac5d..0db8019b2 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -616,21 +616,30 @@ impl IntoIterator for PyDictRef { } } -impl IntoIterator for &PyDictRef { +impl<'a> IntoIterator for &'a PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter; + type IntoIter = DictIterRef<'a>; fn into_iter(self) -> Self::IntoIter { - DictIter::new(self.clone()) + DictIterRef::new(self) } } -impl IntoIterator for &Py { +impl<'a> IntoIterator for &'a Py { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter; + type IntoIter = DictIterRef<'a>; fn into_iter(self) -> Self::IntoIter { - DictIter::new(self.to_owned()) + DictIterRef::new(self) + } +} + +impl<'a> IntoIterator for &'a PyDict { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIterRef<'a>; + + fn into_iter(self) -> Self::IntoIter { + DictIterRef::new(self) } } @@ -655,10 +664,46 @@ impl Iterator for DictIter { } fn size_hint(&self) -> (usize, Option) { - let l = self.dict.entries.len_from_entry_index(self.position); + let l = self.len(); (l, Some(l)) } } +impl ExactSizeIterator for DictIter { + fn len(&self) -> usize { + self.dict.entries.len_from_entry_index(self.position) + } +} + +pub struct DictIterRef<'a> { + dict: &'a PyDict, + position: usize, +} + +impl<'a> DictIterRef<'a> { + pub fn new(dict: &'a PyDict) -> Self { + DictIterRef { dict, position: 0 } + } +} + +impl Iterator for DictIterRef<'_> { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option { + let (position, key, value) = self.dict.entries.next_entry(self.position)?; + self.position = position; + Some((key, value)) + } + + fn size_hint(&self) -> (usize, Option) { + let l = self.len(); + (l, Some(l)) + } +} +impl ExactSizeIterator for DictIterRef<'_> { + fn len(&self) -> usize { + self.dict.entries.len_from_entry_index(self.position) + } +} #[pyclass] trait DictView: PyPayload + PyClassDef + Iterable diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 9a7690fd4..6cb8fb32d 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -643,6 +643,7 @@ impl PyFloat { } // Retrieve inner float value: +#[cfg(feature = "serde")] pub(crate) fn get_value(obj: &PyObject) -> f64 { obj.payload::().unwrap().value } diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 8808eae98..2c41b5224 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -66,8 +66,8 @@ impl PyFunction { ) -> PyResult<()> { let code = &*self.code; let nargs = func_args.args.len(); - let nexpected_args = code.arg_count; - let total_args = code.arg_count + code.kwonlyarg_count; + let nexpected_args = code.arg_count as usize; + let total_args = code.arg_count as usize + code.kwonlyarg_count as usize; // let arg_names = self.code.arg_names(); // This parses the arguments from args and kwargs into @@ -131,7 +131,7 @@ impl PyFunction { // Handle keyword arguments for (name, value) in func_args.kwargs { // Check if we have a parameter with this name: - if let Some(pos) = argpos(code.posonlyarg_count..total_args, &name) { + if let Some(pos) = argpos(code.posonlyarg_count as usize..total_args, &name) { let slot = &mut fastlocals[pos]; if slot.is_some() { return Err(vm.new_type_error(format!( @@ -143,7 +143,7 @@ impl PyFunction { *slot = Some(value); } else if let Some(kwargs) = kwargs.as_ref() { kwargs.set_item(&name, value, vm)?; - } else if argpos(0..code.posonlyarg_count, &name).is_some() { + } else if argpos(0..code.posonlyarg_count as usize, &name).is_some() { posonly_passed_as_kwarg.push(name); } else { return Err(vm.new_type_error(format!( @@ -176,7 +176,7 @@ impl PyFunction { let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice()); let ndefs = defaults.map_or(0, |d| d.len()); - let nrequired = code.arg_count - ndefs; + let nrequired = code.arg_count as usize - ndefs; // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception @@ -244,8 +244,8 @@ impl PyFunction { for (slot, kwarg) in fastlocals .iter_mut() .zip(&*code.varnames) - .skip(code.arg_count) - .take(code.kwonlyarg_count) + .skip(code.arg_count as usize) + .take(code.kwonlyarg_count as usize) .filter(|(slot, _)| slot.is_none()) { if let Some(defaults) = &get_defaults!().1 { diff --git a/vm/src/builtins/function/jitfunc.rs b/vm/src/builtins/function/jitfunc.rs index 4c4bf0976..6b64f40f2 100644 --- a/vm/src/builtins/function/jitfunc.rs +++ b/vm/src/builtins/function/jitfunc.rs @@ -118,7 +118,9 @@ fn get_jit_value(vm: &VirtualMachine, obj: &PyObject) -> Result().unwrap().to_f64(), + )) } else if cls.is(vm.ctx.types.bool_type) { Ok(AbiValue::Bool(bool_::get_value(obj))) } else { @@ -141,7 +143,7 @@ pub(crate) fn get_jit_args<'a>( let nargs = func_args.args.len(); let arg_names = func.code.arg_names(); - if nargs > func.code.arg_count || nargs < func.code.posonlyarg_count { + if nargs > func.code.arg_count as usize || nargs < func.code.posonlyarg_count as usize { return Err(ArgsError::WrongNumberOfArgs); } @@ -160,7 +162,7 @@ pub(crate) fn get_jit_args<'a>( } jit_args.set(arg_idx, get_jit_value(vm, value)?)?; } else if let Some(kwarg_idx) = arg_pos(arg_names.kwonlyargs, name) { - let arg_idx = kwarg_idx + func.code.arg_count; + let arg_idx = kwarg_idx + func.code.arg_count as usize; if jit_args.is_set(arg_idx) { return Err(ArgsError::ArgPassedMultipleTimes); } @@ -175,7 +177,7 @@ pub(crate) fn get_jit_args<'a>( // fill in positional defaults if let Some(defaults) = defaults { for (i, default) in defaults.iter().enumerate() { - let arg_idx = i + func.code.arg_count - defaults.len(); + let arg_idx = i + func.code.arg_count as usize - defaults.len(); if !jit_args.is_set(arg_idx) { jit_args.set(arg_idx, get_jit_value(vm, default)?)?; } @@ -185,7 +187,7 @@ pub(crate) fn get_jit_args<'a>( // fill in keyword only defaults if let Some(kw_only_defaults) = kwdefaults { for (i, name) in arg_names.kwonlyargs.iter().enumerate() { - let arg_idx = i + func.code.arg_count; + let arg_idx = i + func.code.arg_count as usize; if !jit_args.is_set(arg_idx) { let default = kw_only_defaults .get_item(&**name, vm) diff --git a/vm/src/builtins/traceback.rs b/vm/src/builtins/traceback.rs index 0af35f9f5..f645f3ffb 100644 --- a/vm/src/builtins/traceback.rs +++ b/vm/src/builtins/traceback.rs @@ -67,6 +67,7 @@ pub fn init(context: &Context) { PyTraceback::extend_class(context, context.types.traceback_type); } +#[cfg(feature = "serde")] impl serde::Serialize for PyTraceback { fn serialize(&self, s: S) -> Result { use serde::ser::SerializeStruct; diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 3abf70d12..5174301ac 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -19,7 +19,6 @@ use itertools::Itertools; use std::{ collections::HashSet, io::{self, BufRead, BufReader}, - ops::Deref, }; impl std::fmt::Debug for PyBaseException { @@ -958,22 +957,26 @@ fn system_exit_code(exc: PyBaseExceptionRef) -> Option { }) } +#[cfg(feature = "serde")] pub struct SerializeException<'vm, 's> { vm: &'vm VirtualMachine, exc: &'s PyBaseExceptionRef, } +#[cfg(feature = "serde")] impl<'vm, 's> SerializeException<'vm, 's> { pub fn new(vm: &'vm VirtualMachine, exc: &'s PyBaseExceptionRef) -> Self { SerializeException { vm, exc } } } +#[cfg(feature = "serde")] pub struct SerializeExceptionOwned<'vm> { vm: &'vm VirtualMachine, exc: PyBaseExceptionRef, } +#[cfg(feature = "serde")] impl serde::Serialize for SerializeExceptionOwned<'_> { fn serialize(&self, s: S) -> Result { let Self { vm, exc } = self; @@ -981,12 +984,13 @@ impl serde::Serialize for SerializeExceptionOwned<'_> { } } +#[cfg(feature = "serde")] impl serde::Serialize for SerializeException<'_, '_> { fn serialize(&self, s: S) -> Result { use serde::ser::*; let mut struc = s.serialize_struct("PyBaseException", 7)?; - struc.serialize_field("exc_type", self.exc.class().name().deref())?; + struc.serialize_field("exc_type", &*self.exc.class().name())?; let tbs = { struct Tracebacks(PyTracebackRef); impl serde::Serialize for Tracebacks { @@ -1111,8 +1115,6 @@ pub(super) mod types { PyObjectRef, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; - #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] - use std::ops::Deref; // This module is designed to be used as `use builtins::*;`. // Do not add any pub symbols not included in builtins module. @@ -1332,7 +1334,7 @@ pub(super) mod types { // We need this method, because of how `CPython` copies `init` // from `BaseException` in `SimpleExtendsException` macro. // See: `BaseException_new` - if cls.name().deref() == vm.ctx.exceptions.os_error.name().deref() { + if *cls.name() == *vm.ctx.exceptions.os_error.name() { match os_error_optional_new(args.args.to_vec(), vm) { Some(error) => error.to_pyresult(vm), None => PyBaseException::slot_new(cls, args, vm), diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 1c6025a75..868f3a0e2 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -513,10 +513,17 @@ impl ExecutingFrame<'_> { Ok(None) } bytecode::Instruction::ImportName { idx } => { - self.import(vm, Some(self.code.names[idx.get(arg) as usize].to_owned())) + self.import(vm, Some(self.code.names[idx.get(arg) as usize].to_owned()))?; + Ok(None) + } + bytecode::Instruction::ImportNameless => { + self.import(vm, None)?; + Ok(None) + } + bytecode::Instruction::ImportStar => { + self.import_star(vm)?; + Ok(None) } - bytecode::Instruction::ImportNameless => self.import(vm, None), - bytecode::Instruction::ImportStar => self.import_star(vm), bytecode::Instruction::ImportFrom { idx } => { let obj = self.import_from(vm, idx.get(arg))?; self.push_value(obj); @@ -1092,7 +1099,7 @@ impl ExecutingFrame<'_> { } #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn import(&mut self, vm: &VirtualMachine, module: Option) -> FrameResult { + fn import(&mut self, vm: &VirtualMachine, module: Option) -> PyResult<()> { let module = module.unwrap_or_else(|| vm.ctx.empty_str.clone()); let from_list = >>::try_from_object(vm, self.pop_value())?; let level = usize::try_from_object(vm, self.pop_value())?; @@ -1100,7 +1107,7 @@ impl ExecutingFrame<'_> { let module = vm.import(module, from_list, level)?; self.push_value(module); - Ok(None) + Ok(()) } #[cfg_attr(feature = "flame-it", flame("Frame"))] @@ -1127,7 +1134,7 @@ impl ExecutingFrame<'_> { } #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn import_star(&mut self, vm: &VirtualMachine) -> FrameResult { + fn import_star(&mut self, vm: &VirtualMachine) -> PyResult<()> { let module = self.pop_value(); // Grab all the names from the module and put them in the context @@ -1150,7 +1157,7 @@ impl ExecutingFrame<'_> { } } } - Ok(None) + Ok(()) } /// Unwind blocks. diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 0fb964b94..b6b99efcd 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -64,6 +64,7 @@ pub mod object; pub mod prelude; pub mod protocol; pub mod py_io; +#[cfg(feature = "serde")] pub mod py_serde; pub mod readline; pub mod recursion; diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 21db2e40e..a31cf83c6 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -144,8 +144,8 @@ macro_rules! match_class { }; (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { match $obj.payload::<$class>() { - Some($binding) => $expr, - None => $crate::match_class!(match ($obj) { $($rest)* }), + ::std::option::Option::Some($binding) => $expr, + ::std::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), } }; diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 8133117b1..db071243e 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -2,207 +2,90 @@ pub(crate) use decl::make_module; #[pymodule(name = "marshal")] mod decl { + use crate::builtins::code::{CodeObject, Literal, PyObjBag}; + use crate::class::StaticType; use crate::{ builtins::{ - PyBaseExceptionRef, PyByteArray, PyBytes, PyCode, PyDict, PyFloat, PyFrozenSet, PyInt, - PyList, PySet, PyStr, PyTuple, + PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, + PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, }, - bytecode, convert::ToPyObject, function::{ArgBytesLike, OptionalArg}, object::AsObject, protocol::PyBuffer, PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; - use num_bigint::{BigInt, Sign}; + use num_bigint::BigInt; + use num_complex::Complex64; use num_traits::Zero; - - #[repr(u8)] - enum Type { - // Null = b'0', - None = b'N', - False = b'F', - True = b'T', - StopIter = b'S', - Ellipsis = b'.', - Int = b'i', - Float = b'g', - // Complex = b'y', - // Long = b'l', // i32 - Bytes = b's', // = TYPE_STRING - // Interned = b't', - // Ref = b'r', - Tuple = b'(', - List = b'[', - Dict = b'{', - Code = b'c', - Unicode = b'u', - // Unknown = b'?', - Set = b'<', - FrozenSet = b'>', - Ascii = b'a', - // AsciiInterned = b'A', - // SmallTuple = b')', - // ShortAscii = b'z', - // ShortAsciiInterned = b'Z', - } - // const FLAG_REF: u8 = b'\x80'; - - impl TryFrom for Type { - type Error = u8; - fn try_from(value: u8) -> Result { - use Type::*; - Ok(match value { - // b'0' => Null, - b'N' => None, - b'F' => False, - b'T' => True, - b'S' => StopIter, - b'.' => Ellipsis, - b'i' => Int, - b'g' => Float, - // b'y' => Complex, - // b'l' => Long, - b's' => Bytes, - // b't' => Interned, - // b'r' => Ref, - b'(' => Tuple, - b'[' => List, - b'{' => Dict, - b'c' => Code, - b'u' => Unicode, - // b'?' => Unknown, - b'<' => Set, - b'>' => FrozenSet, - b'a' => Ascii, - // b'A' => AsciiInterned, - // b')' => SmallTuple, - // b'z' => ShortAscii, - // b'Z' => ShortAsciiInterned, - c => return Err(c), - }) - } - } + use rustpython_compiler_core::marshal; #[pyattr(name = "version")] - const VERSION: u32 = 4; + use marshal::FORMAT_VERSION; - fn too_short_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "marshal data too short".to_owned(), - ) - } + pub struct DumpError; - /// Dumps a sequence of objects into binary vector. - fn dump_seq( - buf: &mut Vec, - iter: std::slice::Iter, - vm: &VirtualMachine, - ) -> PyResult<()> { - write_size(buf, iter.len(), vm)?; - // For each element, dump into binary, then add its length and value. - for element in iter { - dump_obj(buf, element.clone(), vm)?; - } - Ok(()) - } - - /// Dumping helper function to turn a value into bytes. - fn dump_obj(buf: &mut Vec, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if vm.is_none(&value) { - buf.push(Type::None as u8); - } else if value.is(vm.ctx.exceptions.stop_iteration) { - buf.push(Type::StopIter as u8); - } else if value.is(&vm.ctx.ellipsis) { - buf.push(Type::Ellipsis as u8); - } else { - match_class!(match value { - pyint @ PyInt => { - if pyint.class().is(vm.ctx.types.bool_type) { - let typ = if pyint.as_bigint().is_zero() { - Type::False - } else { - Type::True - }; - buf.push(typ as u8); + impl marshal::Dumpable for PyObjectRef { + type Error = DumpError; + type Constant = Literal; + fn with_dump( + &self, + f: impl FnOnce(marshal::DumpableValue<'_, Self>) -> R, + ) -> Result { + use marshal::DumpableValue::*; + if self.is(PyStopIteration::static_type()) { + return Ok(f(StopIter)); + } + let ret = match_class!(match self { + PyNone => f(None), + PyEllipsis => f(Ellipsis), + ref pyint @ PyInt => { + if self.class().is(PyBool::static_type()) { + f(Boolean(!pyint.as_bigint().is_zero())) } else { - buf.push(Type::Int as u8); - let (sign, int_bytes) = pyint.as_bigint().to_bytes_le(); - let mut len = int_bytes.len() as i32; - if sign == Sign::Minus { - len = -len; - } - buf.extend(len.to_le_bytes()); - buf.extend(int_bytes); + f(Integer(pyint.as_bigint())) } } - pyfloat @ PyFloat => { - buf.push(Type::Float as u8); - buf.extend(pyfloat.to_f64().to_le_bytes()); + ref pyfloat @ PyFloat => { + f(Float(pyfloat.to_f64())) } - pystr @ PyStr => { - buf.push(if pystr.is_ascii() { - Type::Ascii - } else { - Type::Unicode - } as u8); - write_size(buf, pystr.as_str().len(), vm)?; - buf.extend(pystr.as_str().as_bytes()); + ref pycomplex @ PyComplex => { + f(Complex(pycomplex.to_complex64())) } - pylist @ PyList => { - buf.push(Type::List as u8); - let pylist_items = pylist.borrow_vec(); - dump_seq(buf, pylist_items.iter(), vm)?; + ref pystr @ PyStr => { + f(Str(pystr.as_str())) } - pyset @ PySet => { - buf.push(Type::Set as u8); + ref pylist @ PyList => { + f(List(&pylist.borrow_vec())) + } + ref pyset @ PySet => { let elements = pyset.elements(); - dump_seq(buf, elements.iter(), vm)?; + f(Set(&elements)) } - pyfrozen @ PyFrozenSet => { - buf.push(Type::FrozenSet as u8); + ref pyfrozen @ PyFrozenSet => { let elements = pyfrozen.elements(); - dump_seq(buf, elements.iter(), vm)?; + f(Frozenset(&elements)) } - pytuple @ PyTuple => { - buf.push(Type::Tuple as u8); - dump_seq(buf, pytuple.iter(), vm)?; + ref pytuple @ PyTuple => { + f(Tuple(pytuple.as_slice())) } - pydict @ PyDict => { - buf.push(Type::Dict as u8); - write_size(buf, pydict.len(), vm)?; - for (key, value) in pydict { - dump_obj(buf, key, vm)?; - dump_obj(buf, value, vm)?; - } + ref pydict @ PyDict => { + let entries = pydict.into_iter().collect::>(); + f(Dict(&entries)) } - bytes @ PyBytes => { - buf.push(Type::Bytes as u8); - let data = bytes.as_bytes(); - write_size(buf, data.len(), vm)?; - buf.extend(data); + ref bytes @ PyBytes => { + f(Bytes(bytes.as_bytes())) } - bytes @ PyByteArray => { - buf.push(Type::Bytes as u8); - let data = bytes.borrow_buf(); - write_size(buf, data.len(), vm)?; - buf.extend(&*data); + ref bytes @ PyByteArray => { + f(Bytes(&bytes.borrow_buf())) } - co @ PyCode => { - buf.push(Type::Code as u8); - let bytes = co.code.map_clone_bag(&bytecode::BasicBag).to_bytes(); - write_size(buf, bytes.len(), vm)?; - buf.extend(bytes); + ref co @ PyCode => { + f(Code(co)) } - _ => { - return Err(vm.new_not_implemented_error( - "TODO: not implemented yet or marshal unsupported type".to_owned(), - )); - } - }) + _ => return Err(DumpError), + }); + Ok(ret) } - Ok(()) } #[pyfunction] @@ -211,8 +94,16 @@ mod decl { _version: OptionalArg, vm: &VirtualMachine, ) -> PyResult { + use marshal::Dumpable; let mut buf = Vec::new(); - dump_obj(&mut buf, value, vm)?; + value + .with_dump(|val| marshal::serialize_value(&mut buf, val)) + .unwrap_or_else(Err) + .map_err(|DumpError| { + vm.new_not_implemented_error( + "TODO: not implemented yet or marshal unsupported type".to_owned(), + ) + })?; Ok(PyBytes::from(buf)) } @@ -228,38 +119,83 @@ mod decl { Ok(()) } - /// Safely convert usize to 4 le bytes - fn write_size(buf: &mut Vec, x: usize, vm: &VirtualMachine) -> PyResult<()> { - // For marshalling we want to convert lengths to bytes. To save space - // we limit the size to u32 to keep marshalling smaller. - let n = u32::try_from(x).map_err(|_| { - vm.new_value_error("Size exceeds 2^32 capacity for marshaling.".to_owned()) - })?; - buf.extend(n.to_le_bytes()); - Ok(()) - } + #[derive(Copy, Clone)] + struct PyMarshalBag<'a>(&'a VirtualMachine); - /// Read the next 4 bytes of a slice, read as u32, pass as usize. - /// Returns the rest of buffer with the value. - fn read_size<'a>(buf: &'a [u8], vm: &VirtualMachine) -> PyResult<(usize, &'a [u8])> { - if buf.len() < 4 { - return Err(too_short_error(vm)); + impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { + type Value = PyObjectRef; + fn make_bool(&self, value: bool) -> Self::Value { + self.0.ctx.new_bool(value).into() } - let (u32_bytes, rest) = buf.split_at(4); - let length = u32::from_le_bytes(u32_bytes.try_into().unwrap()); - Ok((length as usize, rest)) - } - - /// Reads a list (or tuple) from a buffer. - fn load_seq<'b>(buf: &'b [u8], vm: &VirtualMachine) -> PyResult<(Vec, &'b [u8])> { - let (len, mut buf) = read_size(buf, vm)?; - let mut elements: Vec = Vec::new(); - for _ in 0..len { - let (element, rest) = load_obj(buf, vm)?; - buf = rest; - elements.push(element); + fn make_none(&self) -> Self::Value { + self.0.ctx.none() + } + fn make_ellipsis(&self) -> Self::Value { + self.0.ctx.ellipsis() + } + fn make_float(&self, value: f64) -> Self::Value { + self.0.ctx.new_float(value).into() + } + fn make_complex(&self, value: Complex64) -> Self::Value { + self.0.ctx.new_complex(value).into() + } + fn make_str(&self, value: &str) -> Self::Value { + self.0.ctx.new_str(value).into() + } + fn make_bytes(&self, value: &[u8]) -> Self::Value { + self.0.ctx.new_bytes(value.to_vec()).into() + } + fn make_int(&self, value: BigInt) -> Self::Value { + self.0.ctx.new_int(value).into() + } + fn make_tuple(&self, elements: impl Iterator) -> Self::Value { + self.0.ctx.new_tuple(elements.collect()).into() + } + fn make_code(&self, code: CodeObject) -> Self::Value { + self.0.ctx.new_code(code).into() + } + fn make_stop_iter(&self) -> Result { + Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) + } + fn make_list( + &self, + it: impl Iterator, + ) -> Result { + Ok(self.0.ctx.new_list(it.collect()).into()) + } + fn make_set( + &self, + it: impl Iterator, + ) -> Result { + let vm = self.0; + let set = PySet::new_ref(&vm.ctx); + for elem in it { + set.add(elem, vm).unwrap() + } + Ok(set.into()) + } + fn make_frozenset( + &self, + it: impl Iterator, + ) -> Result { + let vm = self.0; + Ok(PyFrozenSet::from_iter(vm, it).unwrap().to_pyobject(vm)) + } + fn make_dict( + &self, + it: impl Iterator, + ) -> Result { + let vm = self.0; + let dict = vm.ctx.new_dict(); + for (k, v) in it { + dict.set_item(&*k, v, vm).unwrap() + } + Ok(dict.into()) + } + type ConstantBag = PyObjBag<'a>; + fn constant_bag(self) -> Self::ConstantBag { + PyObjBag(&self.0.ctx) } - Ok((elements, buf)) } #[pyfunction] @@ -267,128 +203,21 @@ mod decl { let buf = pybuffer.as_contiguous().ok_or_else(|| { vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous".to_owned()) })?; - let (obj, _) = load_obj(&buf, vm)?; - Ok(obj) - } - - fn load_obj<'b>(buf: &'b [u8], vm: &VirtualMachine) -> PyResult<(PyObjectRef, &'b [u8])> { - let (type_indicator, buf) = buf.split_first().ok_or_else(|| too_short_error(vm))?; - let typ = Type::try_from(*type_indicator) - .map_err(|_| vm.new_value_error("bad marshal data (unknown type code)".to_owned()))?; - let (obj, buf) = match typ { - Type::True => (true.to_pyobject(vm), buf), - Type::False => (false.to_pyobject(vm), buf), - Type::None => (vm.ctx.none(), buf), - Type::StopIter => ( - vm.ctx.exceptions.stop_iteration.to_owned().to_pyobject(vm), - buf, + marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { + marshal::MarshalError::Eof => vm.new_exception_msg( + vm.ctx.exceptions.eof_error.to_owned(), + "marshal data too short".to_owned(), ), - Type::Ellipsis => (vm.ctx.ellipsis(), buf), - Type::Int => { - if buf.len() < 4 { - return Err(too_short_error(vm)); - } - let (len_bytes, buf) = buf.split_at(4); - let len = i32::from_le_bytes(len_bytes.try_into().unwrap()); - let (sign, len) = if len < 0 { - (Sign::Minus, (-len) as usize) - } else { - (Sign::Plus, len as usize) - }; - if buf.len() < len { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(len); - let int = BigInt::from_bytes_le(sign, bytes); - (int.to_pyobject(vm), buf) + marshal::MarshalError::InvalidBytecode => { + vm.new_value_error("Couldn't deserialize python bytecode".to_owned()) } - Type::Float => { - if buf.len() < 8 { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(8); - let number = f64::from_le_bytes(bytes.try_into().unwrap()); - (vm.ctx.new_float(number).into(), buf) + marshal::MarshalError::InvalidUtf8 => { + vm.new_value_error("invalid utf8 in marshalled string".to_owned()) } - Type::Ascii => { - let (len, buf) = read_size(buf, vm)?; - if buf.len() < len { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(len); - let s = String::from_utf8(bytes.to_vec()) - .map_err(|_| vm.new_value_error("invalid utf8 data".to_owned()))?; - (s.to_pyobject(vm), buf) + marshal::MarshalError::BadType => { + vm.new_value_error("bad marshal data (unknown type code)".to_owned()) } - Type::Unicode => { - let (len, buf) = read_size(buf, vm)?; - if buf.len() < len { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(len); - let s = String::from_utf8(bytes.to_vec()) - .map_err(|_| vm.new_value_error("invalid utf8 data".to_owned()))?; - (s.to_pyobject(vm), buf) - } - Type::List => { - let (elements, buf) = load_seq(buf, vm)?; - (vm.ctx.new_list(elements).into(), buf) - } - Type::Set => { - let (elements, buf) = load_seq(buf, vm)?; - let set = PySet::new_ref(&vm.ctx); - for element in elements { - set.add(element, vm)?; - } - (set.to_pyobject(vm), buf) - } - Type::FrozenSet => { - let (elements, buf) = load_seq(buf, vm)?; - let set = PyFrozenSet::from_iter(vm, elements.into_iter())?; - (set.to_pyobject(vm), buf) - } - Type::Tuple => { - let (elements, buf) = load_seq(buf, vm)?; - (vm.ctx.new_tuple(elements).into(), buf) - } - Type::Dict => { - let (len, mut buf) = read_size(buf, vm)?; - let dict = vm.ctx.new_dict(); - for _ in 0..len { - let (key, rest) = load_obj(buf, vm)?; - let (value, rest) = load_obj(rest, vm)?; - buf = rest; - dict.set_item(key.as_object(), value, vm)?; - } - (dict.into(), buf) - } - Type::Bytes => { - // Following CPython, after marshaling, byte arrays are converted into bytes. - let (len, buf) = read_size(buf, vm)?; - if buf.len() < len { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(len); - (vm.ctx.new_bytes(bytes.to_vec()).into(), buf) - } - Type::Code => { - // If prefix is not identifiable, assume CodeObject, error out if it doesn't match. - let (len, buf) = read_size(buf, vm)?; - if buf.len() < len { - return Err(too_short_error(vm)); - } - let (bytes, buf) = buf.split_at(len); - let code = bytecode::CodeObject::from_bytes(bytes).map_err(|e| match e { - bytecode::CodeDeserializeError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "End of file while deserializing bytecode".to_owned(), - ), - _ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()), - })?; - (vm.ctx.new_code(code).into(), buf) - } - }; - Ok((obj, buf)) + }) } #[pyfunction] diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 8a28f903f..04af5b6f2 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -21,7 +21,7 @@ rustpython-parser = { path = "../../compiler/parser" } rustpython-pylib = { path = "../../pylib", default-features = false, optional = true } rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true } # make sure no threading! otherwise wasm build will fail -rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] } +rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings", "serde"] } serde = { workspace = true } From ec7b5861f9486a445fd37b57980e381586ab37b8 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 1 Mar 2023 21:05:59 -0600 Subject: [PATCH 2/2] Address review comments --- compiler/core/src/bytecode.rs | 282 +++++++++++++--------------------- vm/src/builtins/code.rs | 2 + vm/src/builtins/dict.rs | 50 +++--- 3 files changed, 134 insertions(+), 200 deletions(-) diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index fa7146d77..3b3203186 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -218,6 +218,27 @@ impl OpArgType for bool { } } +macro_rules! oparg_enum { + ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $discr:literal,)* }) => { + $(#[$attr])* + $vis enum $name { + $($(#[$var_attr])* $var = $discr,)* + } + + impl OpArgType for $name { + fn to_oparg(self) -> u32 { + self as u32 + } + fn from_oparg(x: u32) -> Option { + Some(match u8::try_from(x).ok()? { + $($discr => Self::$var,)* + _ => return None, + }) + } + } + }; +} + #[derive(Copy, Clone)] pub struct Arg(PhantomData); @@ -292,34 +313,21 @@ impl fmt::Display for Label { } } -/// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u8)] -pub enum ConversionFlag { - /// No conversion - None = 0, // CPython uses -1 but not pleasure for us - /// Converts by calling `str()`. - Str = b's', - /// Converts by calling `ascii()`. - Ascii = b'a', - /// Converts by calling `repr()`. - Repr = b'r', -} - -impl OpArgType for ConversionFlag { - fn to_oparg(self) -> u32 { - self as u32 +oparg_enum!( + /// Transforms a value prior to formatting it. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + #[repr(u8)] + pub enum ConversionFlag { + /// No conversion + None = 0, // CPython uses -1 but not pleasure for us + /// Converts by calling `str()`. + Str = b's', + /// Converts by calling `ascii()`. + Ascii = b'a', + /// Converts by calling `repr()`. + Repr = b'r', } - fn from_oparg(x: u32) -> Option { - Some(match u8::try_from(x).ok()? { - 0 => Self::None, - b's' => Self::Str, - b'a' => Self::Ascii, - b'r' => Self::Repr, - _ => return None, - }) - } -} +); impl TryFrom for ConversionFlag { type Error = usize; @@ -328,28 +336,16 @@ impl TryFrom for ConversionFlag { } } -/// The kind of Raise that occurred. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u8)] -pub enum RaiseKind { - Reraise, - Raise, - RaiseCause, -} - -impl OpArgType for RaiseKind { - fn to_oparg(self) -> u32 { - self as u32 +oparg_enum!( + /// The kind of Raise that occurred. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + #[repr(u8)] + pub enum RaiseKind { + Reraise = 0, + Raise = 1, + RaiseCause = 2, } - fn from_oparg(x: u32) -> Option { - Some(match x { - 0 => Self::Reraise, - 1 => Self::Raise, - 2 => Self::RaiseCause, - _ => return None, - }) - } -} +); pub type NameIdx = u32; @@ -785,138 +781,74 @@ impl BorrowedConstant<'_, C> { } } -/// The possible comparison operators -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum ComparisonOperator { - // be intentional with bits so that we can do eval_ord with just a bitwise and - // bits: | Equal | Greater | Less | - Less = 0b001, - Greater = 0b010, - NotEqual = 0b011, - Equal = 0b100, - LessOrEqual = 0b101, - GreaterOrEqual = 0b110, -} +oparg_enum!( + /// The possible comparison operators + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + #[repr(u8)] + pub enum ComparisonOperator { + // be intentional with bits so that we can do eval_ord with just a bitwise and + // bits: | Equal | Greater | Less | + Less = 0b001, + Greater = 0b010, + NotEqual = 0b011, + Equal = 0b100, + LessOrEqual = 0b101, + GreaterOrEqual = 0b110, + } +); -impl OpArgType for ComparisonOperator { - fn to_oparg(self) -> u32 { - self as u32 +oparg_enum!( + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + #[repr(u8)] + pub enum TestOperator { + In = 0, + NotIn = 1, + Is = 2, + IsNot = 3, + /// two exceptions that match? + ExceptionMatch = 4, } - fn from_oparg(x: u32) -> Option { - Some(match x { - 0b001 => Self::Less, - 0b010 => Self::Greater, - 0b011 => Self::NotEqual, - 0b100 => Self::Equal, - 0b101 => Self::LessOrEqual, - 0b110 => Self::GreaterOrEqual, - _ => return None, - }) - } -} +); -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum TestOperator { - In, - NotIn, - Is, - IsNot, - /// two exceptions that match? - ExceptionMatch, -} +oparg_enum!( + /// The possible Binary operators + /// # Examples + /// + /// ```ignore + /// use rustpython_compiler_core::Instruction::BinaryOperation; + /// use rustpython_compiler_core::BinaryOperator::Add; + /// let op = BinaryOperation {op: Add}; + /// ``` + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + #[repr(u8)] + pub enum BinaryOperator { + Power = 0, + Multiply = 1, + MatrixMultiply = 2, + Divide = 3, + FloorDivide = 4, + Modulo = 5, + Add = 6, + Subtract = 7, + Lshift = 8, + Rshift = 9, + And = 10, + Xor = 11, + Or = 12, + } +); -impl OpArgType for TestOperator { - fn to_oparg(self) -> u32 { - self as u32 +oparg_enum!( + /// The possible unary operators + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + #[repr(u8)] + pub enum UnaryOperator { + Not = 0, + Invert = 1, + Minus = 2, + Plus = 3, } - fn from_oparg(x: u32) -> Option { - Some(match x { - 0 => Self::In, - 1 => Self::NotIn, - 2 => Self::Is, - 3 => Self::IsNot, - 4 => Self::ExceptionMatch, - _ => return None, - }) - } -} - -/// The possible Binary operators -/// # Examples -/// -/// ```ignore -/// use rustpython_compiler_core::Instruction::BinaryOperation; -/// use rustpython_compiler_core::BinaryOperator::Add; -/// let op = BinaryOperation {op: Add}; -/// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum BinaryOperator { - Power, - Multiply, - MatrixMultiply, - Divide, - FloorDivide, - Modulo, - Add, - Subtract, - Lshift, - Rshift, - And, - Xor, - Or, -} - -impl OpArgType for BinaryOperator { - fn to_oparg(self) -> u32 { - self as u32 - } - fn from_oparg(x: u32) -> Option { - Some(match x { - 0 => Self::Power, - 1 => Self::Multiply, - 2 => Self::MatrixMultiply, - 3 => Self::Divide, - 4 => Self::FloorDivide, - 5 => Self::Modulo, - 6 => Self::Add, - 7 => Self::Subtract, - 8 => Self::Lshift, - 9 => Self::Rshift, - 10 => Self::And, - 11 => Self::Xor, - 12 => Self::Or, - _ => return None, - }) - } -} - -/// The possible unary operators -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum UnaryOperator { - Not, - Invert, - Minus, - Plus, -} - -impl OpArgType for UnaryOperator { - fn to_oparg(self) -> u32 { - self as u32 - } - fn from_oparg(x: u32) -> Option { - Some(match x { - 0 => Self::Not, - 1 => Self::Invert, - 2 => Self::Minus, - 3 => Self::Plus, - _ => return None, - }) - } -} +); #[derive(Copy, Clone)] pub struct UnpackExArgs { diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 885e57b4f..fc987ea00 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -79,6 +79,8 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant { } ref t @ super::tuple::PyTuple => { let elements = t.as_slice(); + // SAFETY: Literal is repr(transparent) over PyObjectRef, and a Literal tuple only ever + // has other literals as elements let elements = unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) }; BorrowedConstant::Tuple { elements } } diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index 0db8019b2..5c9c6fcde 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -609,52 +609,52 @@ impl Py { // Implement IntoIterator so that we can easily iterate dictionaries from rust code. impl IntoIterator for PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter; + type IntoIter = DictIntoIter; + + fn into_iter(self) -> Self::IntoIter { + DictIntoIter::new(self) + } +} + +impl<'a> IntoIterator for &'a PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter<'a>; fn into_iter(self) -> Self::IntoIter { DictIter::new(self) } } -impl<'a> IntoIterator for &'a PyDictRef { - type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterRef<'a>; - - fn into_iter(self) -> Self::IntoIter { - DictIterRef::new(self) - } -} - impl<'a> IntoIterator for &'a Py { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterRef<'a>; + type IntoIter = DictIter<'a>; fn into_iter(self) -> Self::IntoIter { - DictIterRef::new(self) + DictIter::new(self) } } impl<'a> IntoIterator for &'a PyDict { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterRef<'a>; + type IntoIter = DictIter<'a>; fn into_iter(self) -> Self::IntoIter { - DictIterRef::new(self) + DictIter::new(self) } } -pub struct DictIter { +pub struct DictIntoIter { dict: PyDictRef, position: usize, } -impl DictIter { - pub fn new(dict: PyDictRef) -> DictIter { - DictIter { dict, position: 0 } +impl DictIntoIter { + pub fn new(dict: PyDictRef) -> DictIntoIter { + DictIntoIter { dict, position: 0 } } } -impl Iterator for DictIter { +impl Iterator for DictIntoIter { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { @@ -668,24 +668,24 @@ impl Iterator for DictIter { (l, Some(l)) } } -impl ExactSizeIterator for DictIter { +impl ExactSizeIterator for DictIntoIter { fn len(&self) -> usize { self.dict.entries.len_from_entry_index(self.position) } } -pub struct DictIterRef<'a> { +pub struct DictIter<'a> { dict: &'a PyDict, position: usize, } -impl<'a> DictIterRef<'a> { +impl<'a> DictIter<'a> { pub fn new(dict: &'a PyDict) -> Self { - DictIterRef { dict, position: 0 } + DictIter { dict, position: 0 } } } -impl Iterator for DictIterRef<'_> { +impl Iterator for DictIter<'_> { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { @@ -699,7 +699,7 @@ impl Iterator for DictIterRef<'_> { (l, Some(l)) } } -impl ExactSizeIterator for DictIterRef<'_> { +impl ExactSizeIterator for DictIter<'_> { fn len(&self) -> usize { self.dict.entries.len_from_entry_index(self.position) }