mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Merge pull request #4524 from coolreader18/no-serde

Browse Source 
Custom marshal enc/decoding impl
This commit is contained in:
Jeong YunWon authored and GitHub committed 2023-03-02 13:10:41 +09:00
2 parents e5cf139b03 ec7b5861f9
commit 22bc2d24ab
29 files changed
+1196 -688
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
-18
View File
@@ -126,15 +126,6 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -1390,7 +1381,6 @@ dependencies = [
"autocfg",
"num-integer",
"num-traits",
"serde",
]
[[package]]
@@ -1400,7 +1390,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"
dependencies = [
"num-traits",
"serde",
]
[[package]]
@@ -1987,7 +1976,6 @@ dependencies = [
"rustpython-ast",
"rustpython-compiler-core",
"rustpython-parser",
"thiserror",
]
[[package]]
@@ -2022,23 +2010,18 @@ dependencies = [
"rustpython-codegen",
"rustpython-compiler-core",
"rustpython-parser",
"thiserror",
]
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
dependencies = [
"bincode",
"bitflags",
"bstr",
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"num_enum",
"serde",
"thiserror",
]
[[package]]
@@ -2108,7 +2091,6 @@ dependencies = [
"rustc-hash",
"rustpython-ast",
"rustpython-compiler-core",
"thiserror",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
compiler/Cargo.toml
-2
View File
@@ -9,5 +9,3 @@ edition = "2021"
rustpython-compiler-core = { path = "core" }
rustpython-codegen = { path = "codegen" }
rustpython-parser = { path = "parser" }
thiserror = { workspace = true }
compiler/codegen/Cargo.toml
+1 -2
View File
@@ -16,9 +16,8 @@ bitflags = { workspace = true }
indexmap = { workspace = true }
itertools = { workspace = true }
log = { workspace = true }
num-complex = { workspace = true, features = ["serde"] }
num-complex = { workspace = true }
num-traits = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
rustpython-parser = { path = "../parser" }
compiler/codegen/src/compile.rs
+10 -8
View File
@@ -249,9 +249,9 @@ impl Compiler {
fn push_output(
&mut self,
flags: bytecode::CodeFlags,
posonlyarg_count: usize,
arg_count: usize,
kwonlyarg_count: usize,
posonlyarg_count: u32,
arg_count: u32,
kwonlyarg_count: u32,
obj_name: String,
) {
let source_path = self.source_path.clone();
@@ -936,9 +936,11 @@ impl Compiler {
self.push_output(
bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
args.posonlyargs.len(),
args.posonlyargs.len() + args.args.len(),
args.kwonlyargs.len(),
args.posonlyargs.len().try_into().unwrap(),
(args.posonlyargs.len() + args.args.len())
.try_into()
.unwrap(),
args.kwonlyargs.len().try_into().unwrap(),
name.to_owned(),
);
@@ -2750,8 +2752,8 @@ impl Compiler {
self.current_source_location = location;
}
fn get_source_line_number(&self) -> usize {
self.current_source_location.row()
fn get_source_line_number(&self) -> u32 {
self.current_source_location.row() as u32
}
fn push_qualified_path(&mut self, name: &str) {
compiler/codegen/src/error.rs
+3 -1
View File
@@ -2,7 +2,7 @@ use std::fmt;
pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
#[derive(Debug, thiserror::Error)]
#[derive(Debug)]
#[non_exhaustive]
pub enum CodegenErrorType {
/// Invalid assignment, cannot store value in target.
@@ -33,6 +33,8 @@ pub enum CodegenErrorType {
NotImplementedYet, // RustPython marker for unimplemented features
}
impl std::error::Error for CodegenErrorType {}
impl fmt::Display for CodegenErrorType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use CodegenErrorType::*;
compiler/codegen/src/ir.rs
+9 -9
View File
@@ -63,11 +63,11 @@ impl Default for Block {
pub struct CodeInfo {
pub flags: CodeFlags,
pub posonlyarg_count: usize, // Number of positional-only arguments
pub arg_count: usize,
pub kwonlyarg_count: usize,
pub posonlyarg_count: u32, // Number of positional-only arguments
pub arg_count: u32,
pub kwonlyarg_count: u32,
pub source_path: String,
pub first_line_number: usize,
pub first_line_number: u32,
pub obj_name: String, // Name of the object that created this code object
pub blocks: Vec<Block>,
@@ -172,15 +172,15 @@ impl CodeInfo {
}
}
fn cell2arg(&self) -> Option<Box<[isize]>> {
fn cell2arg(&self) -> Option<Box<[i32]>> {
if self.cellvar_cache.is_empty() {
return None;
}
let total_args = self.arg_count
+ self.kwonlyarg_count
+ self.flags.contains(CodeFlags::HAS_VARARGS) as usize
+ self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as usize;
+ self.flags.contains(CodeFlags::HAS_VARARGS) as u32
+ self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32;
let mut found_cellarg = false;
let cell2arg = self
@@ -190,10 +190,10 @@ impl CodeInfo {
self.varname_cache
.get_index_of(var)
// check that it's actually an arg
.filter(|i| *i < total_args)
.filter(|i| *i < total_args as usize)
.map_or(-1, |i| {
found_cellarg = true;
i as isize
i as i32
})
})
.collect::<Box<[_]>>();
compiler/core/Cargo.toml
+2 -6
View File
@@ -8,14 +8,10 @@ repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[dependencies]
bincode = { workspace = true }
bitflags = { workspace = true }
bstr = { workspace = true }
itertools = { workspace = true }
num-bigint = { workspace = true, features = ["serde"] }
num-complex = { workspace = true, features = ["serde"] }
num_enum = { workspace = true }
serde = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
num-bigint = { workspace = true }
num-complex = { workspace = true }
lz4_flex = "0.9.2"
compiler/core/src/bytecode.rs
+217 -255
View File
@@ -1,14 +1,12 @@
//! Implement python as a virtual machine with bytecodes. This module
//! implements bytecode structure.
use crate::Location;
use crate::marshal::MarshalError;
use crate::{marshal, Location};
use bitflags::bitflags;
use bstr::ByteSlice;
use itertools::Itertools;
use num_bigint::BigInt;
use num_complex::Complex64;
use num_enum::{IntoPrimitive, TryFromPrimitive};
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;
use std::{collections::BTreeSet, fmt, hash, mem};
@@ -31,9 +29,7 @@ impl Constant for ConstantData {
ConstantData::Str { value } => Str { value },
ConstantData::Bytes { value } => Bytes { value },
ConstantData::Code { code } => Code { code },
ConstantData::Tuple { elements } => Tuple {
elements: Box::new(elements.iter().map(|e| e.borrow_constant())),
},
ConstantData::Tuple { elements } => Tuple { elements },
ConstantData::None => None,
ConstantData::Ellipsis => Ellipsis,
}
@@ -44,6 +40,9 @@ impl Constant for ConstantData {
pub trait ConstantBag: Sized + Copy {
type Constant: Constant;
fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant;
fn make_int(&self, value: BigInt) -> Self::Constant;
fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant;
fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant;
fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name;
}
@@ -55,6 +54,19 @@ impl ConstantBag for BasicBag {
fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant {
constant.to_owned()
}
fn make_int(&self, value: BigInt) -> Self::Constant {
ConstantData::Integer { value }
}
fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant {
ConstantData::Tuple {
elements: elements.collect(),
}
}
fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant {
ConstantData::Code {
code: Box::new(code),
}
}
fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name {
name.to_owned()
}
@@ -62,26 +74,22 @@ impl ConstantBag for BasicBag {
/// Primary container of a single code object. Each python function has
/// a codeobject. Also a module has a codeobject.
#[derive(Clone, Serialize, Deserialize)]
#[derive(Clone)]
pub struct CodeObject<C: Constant = ConstantData> {
pub instructions: Box<[CodeUnit]>,
pub locations: Box<[Location]>,
pub flags: CodeFlags,
pub posonlyarg_count: usize,
pub posonlyarg_count: u32,
// Number of positional-only arguments
pub arg_count: usize,
pub kwonlyarg_count: usize,
pub arg_count: u32,
pub kwonlyarg_count: u32,
pub source_path: C::Name,
pub first_line_number: usize,
pub first_line_number: u32,
pub max_stackdepth: u32,
pub obj_name: C::Name,
// Name of the object that created this code object
pub cell2arg: Option<Box<[isize]>>,
pub cell2arg: Option<Box<[i32]>>,
pub constants: Box<[C]>,
#[serde(bound(
deserialize = "C::Name: serde::Deserialize<'de>",
serialize = "C::Name: serde::Serialize"
))]
pub names: Box<[C::Name]>,
pub varnames: Box<[C::Name]>,
pub cellvars: Box<[C::Name]>,
@@ -89,7 +97,6 @@ pub struct CodeObject<C: Constant = ConstantData> {
}
bitflags! {
#[derive(Serialize, Deserialize)]
pub struct CodeFlags: u16 {
const NEW_LOCALS = 0x01;
const IS_GENERATOR = 0x02;
@@ -114,7 +121,7 @@ impl CodeFlags {
}
/// an opcode argument that may be extended by a prior ExtendedArg
#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Copy, Clone, PartialEq, Eq)]
#[repr(transparent)]
pub struct OpArgByte(pub u8);
impl OpArgByte {
@@ -211,23 +218,28 @@ impl OpArgType for bool {
}
}
macro_rules! enum_oparg {
($t:ident) => {
impl OpArgType for $t {
#[inline(always)]
fn from_oparg(x: u32) -> Option<Self> {
$t::try_from_primitive(x as _).ok()
}
#[inline(always)]
macro_rules! oparg_enum {
($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $discr:literal,)* }) => {
$(#[$attr])*
$vis enum $name {
$($(#[$var_attr])* $var = $discr,)*
}
impl OpArgType for $name {
fn to_oparg(self) -> u32 {
u8::from(self).into()
self as u32
}
fn from_oparg(x: u32) -> Option<Self> {
Some(match u8::try_from(x).ok()? {
$($discr => Self::$var,)*
_ => return None,
})
}
}
};
}
#[derive(Copy, Clone, Serialize, Deserialize)]
#[serde(bound = "")]
#[derive(Copy, Clone)]
pub struct Arg<T: OpArgType>(PhantomData<T>);
impl<T: OpArgType> Arg<T> {
@@ -301,45 +313,45 @@ impl fmt::Display for Label {
}
}
/// Transforms a value prior to formatting it.
#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum ConversionFlag {
/// No conversion
None = 0, // CPython uses -1 but not pleasure for us
/// Converts by calling `str(<value>)`.
Str = b's',
/// Converts by calling `ascii(<value>)`.
Ascii = b'a',
/// Converts by calling `repr(<value>)`.
Repr = b'r',
}
enum_oparg!(ConversionFlag);
oparg_enum!(
/// Transforms a value prior to formatting it.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum ConversionFlag {
/// No conversion
None = 0, // CPython uses -1 but not pleasure for us
/// Converts by calling `str(<value>)`.
Str = b's',
/// Converts by calling `ascii(<value>)`.
Ascii = b'a',
/// Converts by calling `repr(<value>)`.
Repr = b'r',
}
);
impl TryFrom<usize> for ConversionFlag {
type Error = usize;
fn try_from(b: usize) -> Result<Self, Self::Error> {
u8::try_from(b)
.ok()
.and_then(|b| Self::try_from(b).ok())
.ok_or(b)
u32::try_from(b).ok().and_then(Self::from_oparg).ok_or(b)
}
}
/// The kind of Raise that occurred.
#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum RaiseKind {
Reraise,
Raise,
RaiseCause,
}
enum_oparg!(RaiseKind);
oparg_enum!(
/// The kind of Raise that occurred.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum RaiseKind {
Reraise = 0,
Raise = 1,
RaiseCause = 2,
}
);
pub type NameIdx = u32;
/// A Single bytecode instruction.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum Instruction {
/// Importing by name
ImportName {
@@ -562,7 +574,29 @@ pub enum Instruction {
}
const _: () = assert!(mem::size_of::<Instruction>() == 1);
#[derive(Copy, Clone, Serialize, Deserialize)]
impl From<Instruction> for u8 {
#[inline]
fn from(ins: Instruction) -> u8 {
// SAFETY: there's no padding bits
unsafe { std::mem::transmute::<Instruction, u8>(ins) }
}
}
impl TryFrom<u8> for Instruction {
type Error = crate::marshal::MarshalError;
#[inline]
fn try_from(value: u8) -> Result<Self, crate::marshal::MarshalError> {
if value <= u8::from(Instruction::ExtendedArg) {
Ok(unsafe { std::mem::transmute::<u8, Instruction>(value) })
} else {
Err(crate::marshal::MarshalError::InvalidBytecode)
}
}
}
#[derive(Copy, Clone)]
#[repr(C)]
pub struct CodeUnit {
pub op: Instruction,
pub arg: OpArgByte,
@@ -579,7 +613,6 @@ impl CodeUnit {
use self::Instruction::*;
bitflags! {
#[derive(Serialize, Deserialize)]
pub struct MakeFunctionFlags: u8 {
const CLOSURE = 0x01;
const ANNOTATIONS = 0x02;
@@ -607,7 +640,7 @@ impl OpArgType for MakeFunctionFlags {
/// let b = ConstantData::Boolean {value: false};
/// assert_ne!(a, b);
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone)]
pub enum ConstantData {
Tuple { elements: Vec<ConstantData> },
Integer { value: BigInt },
@@ -677,36 +710,40 @@ pub enum BorrowedConstant<'a, C: Constant> {
Str { value: &'a str },
Bytes { value: &'a [u8] },
Code { code: &'a CodeObject<C> },
Tuple { elements: BorrowedTupleIter<'a, C> },
Tuple { elements: &'a [C] },
None,
Ellipsis,
}
type BorrowedTupleIter<'a, C> = Box<dyn Iterator<Item = BorrowedConstant<'a, C>> + 'a>;
impl<C: Constant> Copy for BorrowedConstant<'_, C> {}
impl<C: Constant> Clone for BorrowedConstant<'_, C> {
fn clone(&self) -> Self {
*self
}
}
impl<C: Constant> BorrowedConstant<'_, C> {
// takes `self` because we need to consume the iterator
pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result {
pub fn fmt_display(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
BorrowedConstant::Integer { value } => write!(f, "{value}"),
BorrowedConstant::Float { value } => write!(f, "{value}"),
BorrowedConstant::Complex { value } => write!(f, "{value}"),
BorrowedConstant::Boolean { value } => {
write!(f, "{}", if value { "True" } else { "False" })
write!(f, "{}", if *value { "True" } else { "False" })
}
BorrowedConstant::Str { value } => write!(f, "{value:?}"),
BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()),
BorrowedConstant::Bytes { value } => write!(f, "b\"{}\"", value.escape_ascii()),
BorrowedConstant::Code { code } => write!(f, "{code:?}"),
BorrowedConstant::Tuple { elements } => {
write!(f, "(")?;
let mut first = true;
for c in elements {
for c in *elements {
if first {
first = false
} else {
write!(f, ", ")?;
}
c.fmt_display(f)?;
c.borrow_constant().fmt_display(f)?;
}
write!(f, ")")
}
@@ -733,7 +770,10 @@ impl<C: Constant> BorrowedConstant<'_, C> {
code: Box::new(code.map_clone_bag(&BasicBag)),
},
BorrowedConstant::Tuple { elements } => Tuple {
elements: elements.map(BorrowedConstant::to_owned).collect(),
elements: elements
.iter()
.map(|c| c.borrow_constant().to_owned())
.collect(),
},
BorrowedConstant::None => None,
BorrowedConstant::Ellipsis => Ellipsis,
@@ -741,70 +781,74 @@ impl<C: Constant> BorrowedConstant<'_, C> {
}
}
/// The possible comparison operators
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum ComparisonOperator {
// be intentional with bits so that we can do eval_ord with just a bitwise and
// bits: | Equal | Greater | Less |
Less = 0b001,
Greater = 0b010,
NotEqual = 0b011,
Equal = 0b100,
LessOrEqual = 0b101,
GreaterOrEqual = 0b110,
}
enum_oparg!(ComparisonOperator);
oparg_enum!(
/// The possible comparison operators
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum ComparisonOperator {
// be intentional with bits so that we can do eval_ord with just a bitwise and
// bits: | Equal | Greater | Less |
Less = 0b001,
Greater = 0b010,
NotEqual = 0b011,
Equal = 0b100,
LessOrEqual = 0b101,
GreaterOrEqual = 0b110,
}
);
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum TestOperator {
In,
NotIn,
Is,
IsNot,
/// two exceptions that match?
ExceptionMatch,
}
enum_oparg!(TestOperator);
oparg_enum!(
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum TestOperator {
In = 0,
NotIn = 1,
Is = 2,
IsNot = 3,
/// two exceptions that match?
ExceptionMatch = 4,
}
);
/// The possible Binary operators
/// # Examples
///
/// ```ignore
/// use rustpython_compiler_core::Instruction::BinaryOperation;
/// use rustpython_compiler_core::BinaryOperator::Add;
/// let op = BinaryOperation {op: Add};
/// ```
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum BinaryOperator {
Power,
Multiply,
MatrixMultiply,
Divide,
FloorDivide,
Modulo,
Add,
Subtract,
Lshift,
Rshift,
And,
Xor,
Or,
}
enum_oparg!(BinaryOperator);
oparg_enum!(
/// The possible Binary operators
/// # Examples
///
/// ```ignore
/// use rustpython_compiler_core::Instruction::BinaryOperation;
/// use rustpython_compiler_core::BinaryOperator::Add;
/// let op = BinaryOperation {op: Add};
/// ```
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum BinaryOperator {
Power = 0,
Multiply = 1,
MatrixMultiply = 2,
Divide = 3,
FloorDivide = 4,
Modulo = 5,
Add = 6,
Subtract = 7,
Lshift = 8,
Rshift = 9,
And = 10,
Xor = 11,
Or = 12,
}
);
/// The possible unary operators
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum UnaryOperator {
Not,
Invert,
Minus,
Plus,
}
enum_oparg!(UnaryOperator);
oparg_enum!(
/// The possible unary operators
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum UnaryOperator {
Not = 0,
Invert = 1,
Minus = 2,
Plus = 3,
}
);
#[derive(Copy, Clone)]
pub struct UnpackExArgs {
@@ -867,10 +911,10 @@ impl<C: Constant> CodeObject<C> {
/// Get all arguments of the code object
/// like inspect.getargs
pub fn arg_names(&self) -> Arguments<C::Name> {
let nargs = self.arg_count;
let nkwargs = self.kwonlyarg_count;
let nargs = self.arg_count as usize;
let nkwargs = self.kwonlyarg_count as usize;
let mut varargspos = nargs + nkwargs;
let posonlyargs = &self.varnames[..self.posonlyarg_count];
let posonlyargs = &self.varnames[..self.posonlyarg_count as usize];
let args = &self.varnames[..nargs];
let kwonlyargs = &self.varnames[nargs..varargspos];
@@ -1033,49 +1077,23 @@ impl<C: Constant> CodeObject<C> {
}
}
/// Error that occurs during code deserialization
#[derive(Debug)]
#[non_exhaustive]
pub enum CodeDeserializeError {
/// Unexpected End Of File
Eof,
/// Invalid Bytecode
Other,
}
impl fmt::Display for CodeDeserializeError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::Eof => f.write_str("unexpected end of data"),
Self::Other => f.write_str("invalid bytecode"),
}
}
}
impl std::error::Error for CodeDeserializeError {}
impl CodeObject<ConstantData> {
/// Load a code object from bytes
pub fn from_bytes(data: &[u8]) -> Result<Self, CodeDeserializeError> {
pub fn from_bytes(data: &[u8]) -> Result<Self, MarshalError> {
use lz4_flex::block::DecompressError;
let raw_bincode = lz4_flex::decompress_size_prepended(data).map_err(|e| match e {
DecompressError::OutputTooSmall { .. } | DecompressError::ExpectedAnotherByte => {
CodeDeserializeError::Eof
MarshalError::Eof
}
_ => CodeDeserializeError::Other,
_ => MarshalError::InvalidBytecode,
})?;
let data = bincode::deserialize(&raw_bincode).map_err(|e| match *e {
bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
CodeDeserializeError::Eof
}
_ => CodeDeserializeError::Other,
})?;
Ok(data)
marshal::deserialize_code(&mut &raw_bincode[..], BasicBag)
}
/// Serialize this bytecode to bytes.
pub fn to_bytes(&self) -> Vec<u8> {
let data = bincode::serialize(&self).expect("CodeObject is not serializable");
let mut data = Vec::new();
marshal::serialize_code(&mut data, self);
lz4_flex::compress_prepend_size(&data)
}
}
@@ -1466,7 +1484,7 @@ impl<C: Constant> fmt::Debug for CodeObject<C> {
}
/// A frozen module. Holds a code object and whether it is part of a package
#[derive(Serialize, Deserialize, Debug)]
#[derive(Debug)]
pub struct FrozenModule {
pub code: CodeObject<ConstantData>,
pub package: bool,
@@ -1474,34 +1492,28 @@ pub struct FrozenModule {
pub mod frozen_lib {
use super::*;
use bincode::{options, Options};
use std::io;
use marshal::{Read, Write};
/// Decode a library to a iterable of frozen modules
pub fn decode_lib(bytes: &[u8]) -> FrozenModulesIter {
let data = lz4_flex::decompress_size_prepended(bytes).unwrap();
let r = VecReader { data, pos: 0 };
let mut de = bincode::Deserializer::with_bincode_read(r, options());
let len = u64::deserialize(&mut de).unwrap().try_into().unwrap();
FrozenModulesIter { len, de }
let mut data = marshal::Cursor { data, position: 0 };
let remaining = data.read_u32().unwrap();
FrozenModulesIter { remaining, data }
}
pub struct FrozenModulesIter {
len: usize,
// ideally this could be a SeqAccess, but I think that would require existential types
de: bincode::Deserializer<VecReader, bincode::DefaultOptions>,
remaining: u32,
data: marshal::Cursor<Vec<u8>>,
}
impl Iterator for FrozenModulesIter {
type Item = (String, FrozenModule);
fn next(&mut self) -> Option<Self::Item> {
// manually mimic bincode's seq encoding, which is <len:u64> <element*len>
// This probably won't change (bincode doesn't require padding or anything), but
// it's not guaranteed by semver as far as I can tell
if self.len > 0 {
let entry = Deserialize::deserialize(&mut self.de).unwrap();
self.len -= 1;
if self.remaining > 0 {
let entry = read_entry(&mut self.data).unwrap();
self.remaining -= 1;
Some(entry)
} else {
None
@@ -1509,12 +1521,19 @@ pub mod frozen_lib {
}
fn size_hint(&self) -> (usize, Option<usize>) {
(self.len, Some(self.len))
(self.remaining as usize, Some(self.remaining as usize))
}
}
impl ExactSizeIterator for FrozenModulesIter {}
fn read_entry(rdr: &mut impl Read) -> Result<(String, FrozenModule), marshal::MarshalError> {
let len = rdr.read_u32()?;
let name = rdr.read_str(len)?.to_owned();
let code = marshal::deserialize_code(rdr, BasicBag)?;
let package = rdr.read_u8()? != 0;
Ok((name, FrozenModule { code, package }))
}
/// Encode the given iterator of frozen modules into a compressed vector of bytes
pub fn encode_lib<'a, I>(lib: I) -> Vec<u8>
where
@@ -1522,82 +1541,25 @@ pub mod frozen_lib {
I::IntoIter: ExactSizeIterator + Clone,
{
let iter = lib.into_iter();
let data = options().serialize(&SerializeLib { iter }).unwrap();
let mut data = Vec::new();
write_lib(&mut data, iter);
lz4_flex::compress_prepend_size(&data)
}
struct SerializeLib<I> {
iter: I,
}
impl<'a, I> Serialize for SerializeLib<I>
where
I: ExactSizeIterator<Item = (&'a str, &'a FrozenModule)> + Clone,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.collect_seq(self.iter.clone())
fn write_lib<'a>(
buf: &mut impl Write,
lib: impl ExactSizeIterator<Item = (&'a str, &'a FrozenModule)>,
) {
marshal::write_len(buf, lib.len());
for (name, module) in lib {
write_entry(buf, name, module);
}
}
/// Owned version of bincode::de::read::SliceReader<'a>
struct VecReader {
data: Vec<u8>,
pos: usize,
}
impl io::Read for VecReader {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let mut subslice = &self.data[self.pos..];
let n = io::Read::read(&mut subslice, buf)?;
self.pos += n;
Ok(n)
}
fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
self.get_byte_slice(buf.len())
.map(|data| buf.copy_from_slice(data))
}
}
impl VecReader {
#[inline(always)]
fn get_byte_slice(&mut self, length: usize) -> io::Result<&[u8]> {
let subslice = &self.data[self.pos..];
match subslice.get(..length) {
Some(ret) => {
self.pos += length;
Ok(ret)
}
None => Err(io::ErrorKind::UnexpectedEof.into()),
}
}
}
impl<'storage> bincode::BincodeRead<'storage> for VecReader {
fn forward_read_str<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
where
V: serde::de::Visitor<'storage>,
{
let bytes = self.get_byte_slice(length)?;
match ::std::str::from_utf8(bytes) {
Ok(s) => visitor.visit_str(s),
Err(e) => Err(bincode::ErrorKind::InvalidUtf8Encoding(e).into()),
}
}
fn get_byte_buffer(&mut self, length: usize) -> bincode::Result<Vec<u8>> {
self.get_byte_slice(length)
.map(|x| x.to_vec())
.map_err(Into::into)
}
fn forward_read_bytes<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
where
V: serde::de::Visitor<'storage>,
{
visitor.visit_bytes(self.get_byte_slice(length)?)
}
fn write_entry(buf: &mut impl Write, name: &str, module: &FrozenModule) {
marshal::write_len(buf, name.len());
buf.write_slice(name.as_bytes());
marshal::serialize_code(buf, &module.code);
buf.write_u8(module.package as u8);
}
}
compiler/core/src/error.rs
+16 -2
View File
@@ -1,4 +1,5 @@
use crate::Location;
use std::error::Error as StdError;
use std::fmt::Display;
#[derive(Debug, PartialEq, Eq)]
@@ -16,7 +17,14 @@ impl<T> std::ops::Deref for BaseError<T> {
}
}
impl<T> std::error::Error for BaseError<T> where T: std::fmt::Display + std::fmt::Debug {}
impl<T> StdError for BaseError<T>
where
T: StdError + 'static,
{
fn source(&self) -> Option<&(dyn StdError + 'static)> {
Some(&self.error)
}
}
impl<T> Display for BaseError<T>
where
@@ -51,12 +59,18 @@ impl<T> BaseError<T> {
}
}
#[derive(Debug, thiserror::Error)]
#[derive(Debug)]
pub struct CompileError<T> {
pub body: BaseError<T>,
pub statement: Option<String>,
}
impl<T: StdError + 'static> StdError for CompileError<T> {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
self.body.source()
}
}
impl<T> std::ops::Deref for CompileError<T> {
type Target = BaseError<T>;
fn deref(&self) -> &Self::Target {
compiler/core/src/lib.rs
+1
View File
@@ -4,6 +4,7 @@
mod bytecode;
mod error;
mod location;
pub mod marshal;
mod mode;
pub use bytecode::*;
compiler/core/src/location.rs
+1 -3
View File
@@ -1,7 +1,5 @@
use serde::{Deserialize, Serialize};
/// Sourcecode location.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Location {
pub(super) row: u32,
pub(super) column: u32,
compiler/core/src/marshal.rs
+615
View File
@@ -0,0 +1,615 @@
use core::fmt;
use std::convert::Infallible;
use num_bigint::{BigInt, Sign};
use num_complex::Complex64;
use crate::{bytecode::*, Location};
pub const FORMAT_VERSION: u32 = 4;
#[derive(Debug)]
pub enum MarshalError {
/// Unexpected End Of File
Eof,
/// Invalid Bytecode
InvalidBytecode,
/// Invalid utf8 in string
InvalidUtf8,
/// Bad type marker
BadType,
}
impl fmt::Display for MarshalError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::Eof => f.write_str("unexpected end of data"),
Self::InvalidBytecode => f.write_str("invalid bytecode"),
Self::InvalidUtf8 => f.write_str("invalid utf8"),
Self::BadType => f.write_str("bad type marker"),
}
}
}
impl From<std::str::Utf8Error> for MarshalError {
fn from(_: std::str::Utf8Error) -> Self {
Self::InvalidUtf8
}
}
impl std::error::Error for MarshalError {}
type Result<T, E = MarshalError> = std::result::Result<T, E>;
#[repr(u8)]
enum Type {
// Null = b'0',
None = b'N',
False = b'F',
True = b'T',
StopIter = b'S',
Ellipsis = b'.',
Int = b'i',
Float = b'g',
Complex = b'y',
// Long = b'l', // i32
Bytes = b's', // = TYPE_STRING
// Interned = b't',
// Ref = b'r',
Tuple = b'(',
List = b'[',
Dict = b'{',
Code = b'c',
Unicode = b'u',
// Unknown = b'?',
Set = b'<',
FrozenSet = b'>',
Ascii = b'a',
// AsciiInterned = b'A',
// SmallTuple = b')',
// ShortAscii = b'z',
// ShortAsciiInterned = b'Z',
}
// const FLAG_REF: u8 = b'\x80';
impl TryFrom<u8> for Type {
type Error = MarshalError;
fn try_from(value: u8) -> Result<Self> {
use Type::*;
Ok(match value {
// b'0' => Null,
b'N' => None,
b'F' => False,
b'T' => True,
b'S' => StopIter,
b'.' => Ellipsis,
b'i' => Int,
b'g' => Float,
b'y' => Complex,
// b'l' => Long,
b's' => Bytes,
// b't' => Interned,
// b'r' => Ref,
b'(' => Tuple,
b'[' => List,
b'{' => Dict,
b'c' => Code,
b'u' => Unicode,
// b'?' => Unknown,
b'<' => Set,
b'>' => FrozenSet,
b'a' => Ascii,
// b'A' => AsciiInterned,
// b')' => SmallTuple,
// b'z' => ShortAscii,
// b'Z' => ShortAsciiInterned,
_ => return Err(MarshalError::BadType),
})
}
}
pub trait Read {
fn read_slice(&mut self, n: u32) -> Result<&[u8]>;
fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> {
self.read_slice(N as u32).map(|s| s.try_into().unwrap())
}
fn read_str(&mut self, len: u32) -> Result<&str> {
Ok(std::str::from_utf8(self.read_slice(len)?)?)
}
fn read_u8(&mut self) -> Result<u8> {
Ok(u8::from_le_bytes(*self.read_array()?))
}
fn read_u16(&mut self) -> Result<u16> {
Ok(u16::from_le_bytes(*self.read_array()?))
}
fn read_u32(&mut self) -> Result<u32> {
Ok(u32::from_le_bytes(*self.read_array()?))
}
fn read_u64(&mut self) -> Result<u64> {
Ok(u64::from_le_bytes(*self.read_array()?))
}
}
impl Read for &[u8] {
fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
let data = self.get(..n as usize).ok_or(MarshalError::Eof)?;
*self = &self[n as usize..];
Ok(data)
}
}
pub struct Cursor<B> {
pub data: B,
pub position: usize,
}
impl<B: AsRef<[u8]>> Read for Cursor<B> {
fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
let data = &self.data.as_ref()[self.position..];
let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?;
self.position += n as usize;
Ok(slice)
}
}
pub fn deserialize_code<R: Read, Bag: ConstantBag>(
rdr: &mut R,
bag: Bag,
) -> Result<CodeObject<Bag::Constant>> {
let len = rdr.read_u32()?;
let instructions = rdr.read_slice(len * 2)?;
let instructions = instructions
.chunks_exact(2)
.map(|cu| {
let op = Instruction::try_from(cu[0])?;
let arg = OpArgByte(cu[1]);
Ok(CodeUnit { op, arg })
})
.collect::<Result<Box<[CodeUnit]>>>()?;
let len = rdr.read_u32()?;
let locations = (0..len)
.map(|_| {
Ok(Location {
row: rdr.read_u32()?,
column: rdr.read_u32()?,
})
})
.collect::<Result<Box<[Location]>>>()?;
let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?);
let posonlyarg_count = rdr.read_u32()?;
let arg_count = rdr.read_u32()?;
let kwonlyarg_count = rdr.read_u32()?;
let len = rdr.read_u32()?;
let source_path = bag.make_name(rdr.read_str(len)?);
let first_line_number = rdr.read_u32()?;
let max_stackdepth = rdr.read_u32()?;
let len = rdr.read_u32()?;
let obj_name = bag.make_name(rdr.read_str(len)?);
let len = rdr.read_u32()?;
let cell2arg = (len != 0)
.then(|| {
(0..len)
.map(|_| Ok(rdr.read_u32()? as i32))
.collect::<Result<Box<[i32]>>>()
})
.transpose()?;
let len = rdr.read_u32()?;
let constants = (0..len)
.map(|_| deserialize_value(rdr, bag))
.collect::<Result<Box<[_]>>>()?;
let mut read_names = || {
let len = rdr.read_u32()?;
(0..len)
.map(|_| {
let len = rdr.read_u32()?;
Ok(bag.make_name(rdr.read_str(len)?))
})
.collect::<Result<Box<[_]>>>()
};
let names = read_names()?;
let varnames = read_names()?;
let cellvars = read_names()?;
let freevars = read_names()?;
Ok(CodeObject {
instructions,
locations,
flags,
posonlyarg_count,
arg_count,
kwonlyarg_count,
source_path,
first_line_number,
max_stackdepth,
obj_name,
cell2arg,
constants,
names,
varnames,
cellvars,
freevars,
})
}
pub trait MarshalBag: Copy {
type Value;
fn make_bool(&self, value: bool) -> Self::Value;
fn make_none(&self) -> Self::Value;
fn make_ellipsis(&self) -> Self::Value;
fn make_float(&self, value: f64) -> Self::Value;
fn make_complex(&self, value: Complex64) -> Self::Value;
fn make_str(&self, value: &str) -> Self::Value;
fn make_bytes(&self, value: &[u8]) -> Self::Value;
fn make_int(&self, value: BigInt) -> Self::Value;
fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value;
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value;
fn make_stop_iter(&self) -> Result<Self::Value>;
fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_set(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_dict(
&self,
it: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value>;
type ConstantBag: ConstantBag;
fn constant_bag(self) -> Self::ConstantBag;
}
impl<Bag: ConstantBag> MarshalBag for Bag {
type Value = Bag::Constant;
fn make_bool(&self, value: bool) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Boolean { value })
}
fn make_none(&self) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::None)
}
fn make_ellipsis(&self) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Ellipsis)
}
fn make_float(&self, value: f64) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Float { value })
}
fn make_complex(&self, value: Complex64) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Complex { value })
}
fn make_str(&self, value: &str) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Str { value })
}
fn make_bytes(&self, value: &[u8]) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Bytes { value })
}
fn make_int(&self, value: BigInt) -> Self::Value {
self.make_int(value)
}
fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value {
self.make_tuple(elements)
}
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value {
self.make_code(code)
}
fn make_stop_iter(&self) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_list(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_set(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_dict(
&self,
_: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
type ConstantBag = Self;
fn constant_bag(self) -> Self::ConstantBag {
self
}
}
pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Result<Bag::Value> {
let typ = Type::try_from(rdr.read_u8()?)?;
let value = match typ {
Type::True => bag.make_bool(true),
Type::False => bag.make_bool(false),
Type::None => bag.make_none(),
Type::StopIter => bag.make_stop_iter()?,
Type::Ellipsis => bag.make_ellipsis(),
Type::Int => {
let len = rdr.read_u32()? as i32;
let sign = if len < 0 { Sign::Minus } else { Sign::Plus };
let bytes = rdr.read_slice(len.unsigned_abs())?;
let int = BigInt::from_bytes_le(sign, bytes);
bag.make_int(int)
}
Type::Float => {
let value = f64::from_bits(rdr.read_u64()?);
bag.make_float(value)
}
Type::Complex => {
let re = f64::from_bits(rdr.read_u64()?);
let im = f64::from_bits(rdr.read_u64()?);
let value = Complex64 { re, im };
bag.make_complex(value)
}
Type::Ascii | Type::Unicode => {
let len = rdr.read_u32()?;
let value = rdr.read_str(len)?;
bag.make_str(value)
}
Type::Tuple => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_tuple(it))?
}
Type::List => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_list(it))??
}
Type::Set => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_set(it))??
}
Type::FrozenSet => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_frozenset(it))??
}
Type::Dict => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| {
let k = deserialize_value(rdr, bag)?;
let v = deserialize_value(rdr, bag)?;
Ok::<_, MarshalError>((k, v))
});
itertools::process_results(it, |it| bag.make_dict(it))??
}
Type::Bytes => {
// Following CPython, after marshaling, byte arrays are converted into bytes.
let len = rdr.read_u32()?;
let value = rdr.read_slice(len)?;
bag.make_bytes(value)
}
Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?),
};
Ok(value)
}
pub trait Dumpable: Sized {
type Error;
type Constant: Constant;
fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error>;
}
pub enum DumpableValue<'a, D: Dumpable> {
Integer(&'a BigInt),
Float(f64),
Complex(Complex64),
Boolean(bool),
Str(&'a str),
Bytes(&'a [u8]),
Code(&'a CodeObject<D::Constant>),
Tuple(&'a [D]),
None,
Ellipsis,
StopIter,
List(&'a [D]),
Set(&'a [D]),
Frozenset(&'a [D]),
Dict(&'a [(D, D)]),
}
impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> {
fn from(c: BorrowedConstant<'a, C>) -> Self {
match c {
BorrowedConstant::Integer { value } => Self::Integer(value),
BorrowedConstant::Float { value } => Self::Float(value),
BorrowedConstant::Complex { value } => Self::Complex(value),
BorrowedConstant::Boolean { value } => Self::Boolean(value),
BorrowedConstant::Str { value } => Self::Str(value),
BorrowedConstant::Bytes { value } => Self::Bytes(value),
BorrowedConstant::Code { code } => Self::Code(code),
BorrowedConstant::Tuple { elements } => Self::Tuple(elements),
BorrowedConstant::None => Self::None,
BorrowedConstant::Ellipsis => Self::Ellipsis,
}
}
}
impl<C: Constant> Dumpable for C {
type Error = Infallible;
type Constant = Self;
#[inline(always)]
fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error> {
Ok(f(self.borrow_constant().into()))
}
}
pub trait Write {
fn write_slice(&mut self, slice: &[u8]);
fn write_u8(&mut self, v: u8) {
self.write_slice(&v.to_le_bytes())
}
fn write_u16(&mut self, v: u16) {
self.write_slice(&v.to_le_bytes())
}
fn write_u32(&mut self, v: u32) {
self.write_slice(&v.to_le_bytes())
}
fn write_u64(&mut self, v: u64) {
self.write_slice(&v.to_le_bytes())
}
}
impl Write for Vec<u8> {
fn write_slice(&mut self, slice: &[u8]) {
self.extend_from_slice(slice)
}
}
pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) {
let Ok(len) = len.try_into() else { panic!("too long to serialize") };
buf.write_u32(len);
}
pub fn serialize_value<W: Write, D: Dumpable>(
buf: &mut W,
constant: DumpableValue<'_, D>,
) -> Result<(), D::Error> {
match constant {
DumpableValue::Integer(int) => {
buf.write_u8(Type::Int as u8);
let (sign, bytes) = int.to_bytes_le();
let len: i32 = bytes.len().try_into().expect("too long to serialize");
let len = if sign == Sign::Minus { -len } else { len };
buf.write_u32(len as u32);
buf.write_slice(&bytes);
}
DumpableValue::Float(f) => {
buf.write_u8(Type::Float as u8);
buf.write_u64(f.to_bits());
}
DumpableValue::Complex(c) => {
buf.write_u8(Type::Complex as u8);
buf.write_u64(c.re.to_bits());
buf.write_u64(c.im.to_bits());
}
DumpableValue::Boolean(b) => {
buf.write_u8(if b { Type::True } else { Type::False } as u8);
}
DumpableValue::Str(s) => {
buf.write_u8(Type::Unicode as u8);
write_len(buf, s.len());
buf.write_slice(s.as_bytes());
}
DumpableValue::Bytes(b) => {
buf.write_u8(Type::Bytes as u8);
write_len(buf, b.len());
buf.write_slice(b);
}
DumpableValue::Code(c) => {
buf.write_u8(Type::Code as u8);
serialize_code(buf, c);
}
DumpableValue::Tuple(tup) => {
buf.write_u8(Type::Tuple as u8);
write_len(buf, tup.len());
for val in tup {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::None => {
buf.write_u8(Type::None as u8);
}
DumpableValue::Ellipsis => {
buf.write_u8(Type::Ellipsis as u8);
}
DumpableValue::StopIter => {
buf.write_u8(Type::StopIter as u8);
}
DumpableValue::List(l) => {
buf.write_u8(Type::List as u8);
write_len(buf, l.len());
for val in l {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Set(set) => {
buf.write_u8(Type::Set as u8);
write_len(buf, set.len());
for val in set {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Frozenset(set) => {
buf.write_u8(Type::FrozenSet as u8);
write_len(buf, set.len());
for val in set {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Dict(d) => {
buf.write_u8(Type::Dict as u8);
write_len(buf, d.len());
for (k, v) in d {
k.with_dump(|val| serialize_value(buf, val))??;
v.with_dump(|val| serialize_value(buf, val))??;
}
}
}
Ok(())
}
pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) {
write_len(buf, code.instructions.len());
// SAFETY: it's ok to transmute CodeUnit to [u8; 2]
let (_, instructions_bytes, _) = unsafe { code.instructions.align_to() };
buf.write_slice(instructions_bytes);
write_len(buf, code.locations.len());
for loc in &*code.locations {
buf.write_u32(loc.row);
buf.write_u32(loc.column);
}
buf.write_u16(code.flags.bits());
buf.write_u32(code.posonlyarg_count);
buf.write_u32(code.arg_count);
buf.write_u32(code.kwonlyarg_count);
write_len(buf, code.source_path.as_ref().len());
buf.write_slice(code.source_path.as_ref().as_bytes());
buf.write_u32(code.first_line_number);
buf.write_u32(code.max_stackdepth);
write_len(buf, code.obj_name.as_ref().len());
buf.write_slice(code.obj_name.as_ref().as_bytes());
let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]);
write_len(buf, cell2arg.len());
for &i in cell2arg {
buf.write_u32(i as u32)
}
write_len(buf, code.constants.len());
for constant in &*code.constants {
serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {})
}
let mut write_names = |names: &[C::Name]| {
write_len(buf, names.len());
for name in names {
write_len(buf, name.as_ref().len());
buf.write_slice(name.as_ref().as_bytes());
}
};
write_names(&code.names);
write_names(&code.varnames);
write_names(&code.cellvars);
write_names(&code.freevars);
}
compiler/parser/Cargo.toml
-1
View File
@@ -26,7 +26,6 @@ itertools = { workspace = true }
log = { workspace = true }
num-bigint = { workspace = true }
num-traits = { workspace = true }
thiserror = { workspace = true }
unicode_names2 = { workspace = true }
unic-emoji-char = "0.9.0"
compiler/parser/src/parser.rs
+3 -1
View File
@@ -200,7 +200,7 @@ pub fn parse_tokens(
pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
/// Represents the different types of errors that can occur during parsing.
#[derive(Debug, PartialEq, thiserror::Error)]
#[derive(Debug, PartialEq)]
pub enum ParseErrorType {
/// Parser encountered an unexpected end of input
Eof,
@@ -215,6 +215,8 @@ pub enum ParseErrorType {
Lexical(LexicalErrorType),
}
impl std::error::Error for ParseErrorType {}
// Convert `lalrpop_util::ParseError` to our internal type
fn parse_error_from_lalrpop(
err: LalrpopError<Location, Tok, LexicalError>,
compiler/src/lib.rs
+33 -5
View File
@@ -7,12 +7,40 @@ use rustpython_parser::{
pub use rustpython_codegen::compile::CompileOpts;
pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode};
#[derive(Debug, thiserror::Error)]
use std::error::Error as StdError;
use std::fmt;
#[derive(Debug)]
pub enum CompileErrorType {
#[error(transparent)]
Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
#[error(transparent)]
Parse(#[from] parser::ParseErrorType),
Codegen(rustpython_codegen::error::CodegenErrorType),
Parse(parser::ParseErrorType),
}
impl StdError for CompileErrorType {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
match self {
CompileErrorType::Codegen(e) => e.source(),
CompileErrorType::Parse(e) => e.source(),
}
}
}
impl fmt::Display for CompileErrorType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
CompileErrorType::Codegen(e) => e.fmt(f),
CompileErrorType::Parse(e) => e.fmt(f),
}
}
}
impl From<rustpython_codegen::error::CodegenErrorType> for CompileErrorType {
fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self {
CompileErrorType::Codegen(source)
}
}
impl From<parser::ParseErrorType> for CompileErrorType {
fn from(source: parser::ParseErrorType) -> Self {
CompileErrorType::Parse(source)
}
}
pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
vm/Cargo.toml
+4 -3
View File
@@ -21,6 +21,7 @@ compiler = ["parser", "codegen", "rustpython-compiler"]
ast = ["rustpython-ast"]
codegen = ["rustpython-codegen", "ast"]
parser = ["rustpython-parser", "ast"]
serde = ["dep:serde"]
[dependencies]
rustpython-compiler = { path = "../compiler", optional = true, version = "0.2.0" }
@@ -47,8 +48,8 @@ itertools = { workspace = true }
libc = { workspace = true }
log = { workspace = true }
nix = { workspace = true }
num-bigint = { workspace = true, features = ["serde"] }
num-complex = { workspace = true, features = ["serde"] }
num-bigint = { workspace = true }
num-complex = { workspace = true }
num-integer = { workspace = true }
num-rational = { workspace = true }
num-traits = { workspace = true }
@@ -57,7 +58,7 @@ once_cell = { workspace = true }
parking_lot = { workspace = true }
paste = { workspace = true }
rand = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde = { workspace = true, optional = true }
static_assertions = { workspace = true }
thiserror = { workspace = true }
thread_local = { workspace = true }
vm/src/builtins/code.rs
+28 -13
View File
@@ -17,15 +17,15 @@ use std::{borrow::Borrow, fmt, ops::Deref};
#[derive(FromArgs)]
pub struct ReplaceArgs {
#[pyarg(named, optional)]
co_posonlyargcount: OptionalArg<usize>,
co_posonlyargcount: OptionalArg<u32>,
#[pyarg(named, optional)]
co_argcount: OptionalArg<usize>,
co_argcount: OptionalArg<u32>,
#[pyarg(named, optional)]
co_kwonlyargcount: OptionalArg<usize>,
co_kwonlyargcount: OptionalArg<u32>,
#[pyarg(named, optional)]
co_filename: OptionalArg<PyStrRef>,
#[pyarg(named, optional)]
co_firstlineno: OptionalArg<usize>,
co_firstlineno: OptionalArg<u32>,
#[pyarg(named, optional)]
co_consts: OptionalArg<Vec<PyObjectRef>>,
#[pyarg(named, optional)]
@@ -39,6 +39,7 @@ pub struct ReplaceArgs {
}
#[derive(Clone)]
#[repr(transparent)]
pub struct Literal(PyObjectRef);
impl Borrow<PyObject> for Literal {
@@ -77,9 +78,11 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<Literal> {
BorrowedConstant::Code { code: &c.code }
}
ref t @ super::tuple::PyTuple => {
BorrowedConstant::Tuple {
elements: Box::new(t.iter().map(|o| borrow_obj_constant(o))),
}
let elements = t.as_slice();
// SAFETY: Literal is repr(transparent) over PyObjectRef, and a Literal tuple only ever
// has other literals as elements
let elements = unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) };
BorrowedConstant::Tuple { elements }
}
super::singletons::PyNone => BorrowedConstant::None,
super::slice::PyEllipsis => BorrowedConstant::Ellipsis,
@@ -117,8 +120,8 @@ impl ConstantBag for PyObjBag<'_> {
}
bytecode::BorrowedConstant::Tuple { elements } => {
let elements = elements
.into_iter()
.map(|constant| self.make_constant(constant).0)
.iter()
.map(|constant| self.make_constant(constant.borrow_constant()).0)
.collect();
ctx.new_tuple(elements).into()
}
@@ -131,6 +134,18 @@ impl ConstantBag for PyObjBag<'_> {
fn make_name(&self, name: &str) -> &'static PyStrInterned {
self.0.intern_str(name)
}
fn make_int(&self, value: num_bigint::BigInt) -> Self::Constant {
Literal(self.0.new_int(value).into())
}
fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant {
Literal(self.0.new_tuple(elements.map(|lit| lit.0).collect()).into())
}
fn make_code(&self, code: CodeObject) -> Self::Constant {
Literal(self.0.new_code(code).into())
}
}
pub type CodeObject = bytecode::CodeObject<Literal>;
@@ -205,12 +220,12 @@ impl PyRef<PyCode> {
#[pygetset]
fn co_posonlyargcount(self) -> usize {
self.code.posonlyarg_count
self.code.posonlyarg_count as usize
}
#[pygetset]
fn co_argcount(self) -> usize {
self.code.arg_count
self.code.arg_count as usize
}
#[pygetset]
@@ -237,12 +252,12 @@ impl PyRef<PyCode> {
#[pygetset]
fn co_firstlineno(self) -> usize {
self.code.first_line_number
self.code.first_line_number as usize
}
#[pygetset]
fn co_kwonlyargcount(self) -> usize {
self.code.kwonlyarg_count
self.code.kwonlyarg_count as usize
}
#[pygetset]
vm/src/builtins/complex.rs
+6
View File
@@ -28,6 +28,12 @@ pub struct PyComplex {
value: Complex64,
}
impl PyComplex {
pub fn to_complex64(self) -> Complex64 {
self.value
}
}
impl PyPayload for PyComplex {
fn class(vm: &VirtualMachine) -> &'static Py<PyType> {
vm.ctx.types.complex_type
vm/src/builtins/dict.rs
+58 -13
View File
@@ -609,43 +609,52 @@ impl Py<PyDict> {
// Implement IntoIterator so that we can easily iterate dictionaries from rust code.
impl IntoIterator for PyDictRef {
type Item = (PyObjectRef, PyObjectRef);
type IntoIter = DictIter;
type IntoIter = DictIntoIter;
fn into_iter(self) -> Self::IntoIter {
DictIntoIter::new(self)
}
}
impl<'a> IntoIterator for &'a PyDictRef {
type Item = (PyObjectRef, PyObjectRef);
type IntoIter = DictIter<'a>;
fn into_iter(self) -> Self::IntoIter {
DictIter::new(self)
}
}
impl IntoIterator for &PyDictRef {
impl<'a> IntoIterator for &'a Py<PyDict> {
type Item = (PyObjectRef, PyObjectRef);
type IntoIter = DictIter;
type IntoIter = DictIter<'a>;
fn into_iter(self) -> Self::IntoIter {
DictIter::new(self.clone())
DictIter::new(self)
}
}
impl IntoIterator for &Py<PyDict> {
impl<'a> IntoIterator for &'a PyDict {
type Item = (PyObjectRef, PyObjectRef);
type IntoIter = DictIter;
type IntoIter = DictIter<'a>;
fn into_iter(self) -> Self::IntoIter {
DictIter::new(self.to_owned())
DictIter::new(self)
}
}
pub struct DictIter {
pub struct DictIntoIter {
dict: PyDictRef,
position: usize,
}
impl DictIter {
pub fn new(dict: PyDictRef) -> DictIter {
DictIter { dict, position: 0 }
impl DictIntoIter {
pub fn new(dict: PyDictRef) -> DictIntoIter {
DictIntoIter { dict, position: 0 }
}
}
impl Iterator for DictIter {
impl Iterator for DictIntoIter {
type Item = (PyObjectRef, PyObjectRef);
fn next(&mut self) -> Option<Self::Item> {
@@ -655,10 +664,46 @@ impl Iterator for DictIter {
}
fn size_hint(&self) -> (usize, Option<usize>) {
let l = self.dict.entries.len_from_entry_index(self.position);
let l = self.len();
(l, Some(l))
}
}
impl ExactSizeIterator for DictIntoIter {
fn len(&self) -> usize {
self.dict.entries.len_from_entry_index(self.position)
}
}
pub struct DictIter<'a> {
dict: &'a PyDict,
position: usize,
}
impl<'a> DictIter<'a> {
pub fn new(dict: &'a PyDict) -> Self {
DictIter { dict, position: 0 }
}
}
impl Iterator for DictIter<'_> {
type Item = (PyObjectRef, PyObjectRef);
fn next(&mut self) -> Option<Self::Item> {
let (position, key, value) = self.dict.entries.next_entry(self.position)?;
self.position = position;
Some((key, value))
}
fn size_hint(&self) -> (usize, Option<usize>) {
let l = self.len();
(l, Some(l))
}
}
impl ExactSizeIterator for DictIter<'_> {
fn len(&self) -> usize {
self.dict.entries.len_from_entry_index(self.position)
}
}
#[pyclass]
trait DictView: PyPayload + PyClassDef + Iterable
vm/src/builtins/float.rs
+1
View File
@@ -643,6 +643,7 @@ impl PyFloat {
}
// Retrieve inner float value:
#[cfg(feature = "serde")]
pub(crate) fn get_value(obj: &PyObject) -> f64 {
obj.payload::<PyFloat>().unwrap().value
}
vm/src/builtins/function.rs
+7 -7
View File
@@ -66,8 +66,8 @@ impl PyFunction {
) -> PyResult<()> {
let code = &*self.code;
let nargs = func_args.args.len();
let nexpected_args = code.arg_count;
let total_args = code.arg_count + code.kwonlyarg_count;
let nexpected_args = code.arg_count as usize;
let total_args = code.arg_count as usize + code.kwonlyarg_count as usize;
// let arg_names = self.code.arg_names();
// This parses the arguments from args and kwargs into
@@ -131,7 +131,7 @@ impl PyFunction {
// Handle keyword arguments
for (name, value) in func_args.kwargs {
// Check if we have a parameter with this name:
if let Some(pos) = argpos(code.posonlyarg_count..total_args, &name) {
if let Some(pos) = argpos(code.posonlyarg_count as usize..total_args, &name) {
let slot = &mut fastlocals[pos];
if slot.is_some() {
return Err(vm.new_type_error(format!(
@@ -143,7 +143,7 @@ impl PyFunction {
*slot = Some(value);
} else if let Some(kwargs) = kwargs.as_ref() {
kwargs.set_item(&name, value, vm)?;
} else if argpos(0..code.posonlyarg_count, &name).is_some() {
} else if argpos(0..code.posonlyarg_count as usize, &name).is_some() {
posonly_passed_as_kwarg.push(name);
} else {
return Err(vm.new_type_error(format!(
@@ -176,7 +176,7 @@ impl PyFunction {
let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice());
let ndefs = defaults.map_or(0, |d| d.len());
let nrequired = code.arg_count - ndefs;
let nrequired = code.arg_count as usize - ndefs;
// Given the number of defaults available, check all the arguments for which we
// _don't_ have defaults; if any are missing, raise an exception
@@ -244,8 +244,8 @@ impl PyFunction {
for (slot, kwarg) in fastlocals
.iter_mut()
.zip(&*code.varnames)
.skip(code.arg_count)
.take(code.kwonlyarg_count)
.skip(code.arg_count as usize)
.take(code.kwonlyarg_count as usize)
.filter(|(slot, _)| slot.is_none())
{
if let Some(defaults) = &get_defaults!().1 {
vm/src/builtins/function/jitfunc.rs
+7 -5
View File
@@ -118,7 +118,9 @@ fn get_jit_value(vm: &VirtualMachine, obj: &PyObject) -> Result<AbiValue, ArgsEr
.map(AbiValue::Int)
.ok_or(ArgsError::IntOverflow)
} else if cls.is(vm.ctx.types.float_type) {
Ok(AbiValue::Float(float::get_value(obj)))
Ok(AbiValue::Float(
obj.downcast_ref::<float::PyFloat>().unwrap().to_f64(),
))
} else if cls.is(vm.ctx.types.bool_type) {
Ok(AbiValue::Bool(bool_::get_value(obj)))
} else {
@@ -141,7 +143,7 @@ pub(crate) fn get_jit_args<'a>(
let nargs = func_args.args.len();
let arg_names = func.code.arg_names();
if nargs > func.code.arg_count || nargs < func.code.posonlyarg_count {
if nargs > func.code.arg_count as usize || nargs < func.code.posonlyarg_count as usize {
return Err(ArgsError::WrongNumberOfArgs);
}
@@ -160,7 +162,7 @@ pub(crate) fn get_jit_args<'a>(
}
jit_args.set(arg_idx, get_jit_value(vm, value)?)?;
} else if let Some(kwarg_idx) = arg_pos(arg_names.kwonlyargs, name) {
let arg_idx = kwarg_idx + func.code.arg_count;
let arg_idx = kwarg_idx + func.code.arg_count as usize;
if jit_args.is_set(arg_idx) {
return Err(ArgsError::ArgPassedMultipleTimes);
}
@@ -175,7 +177,7 @@ pub(crate) fn get_jit_args<'a>(
// fill in positional defaults
if let Some(defaults) = defaults {
for (i, default) in defaults.iter().enumerate() {
let arg_idx = i + func.code.arg_count - defaults.len();
let arg_idx = i + func.code.arg_count as usize - defaults.len();
if !jit_args.is_set(arg_idx) {
jit_args.set(arg_idx, get_jit_value(vm, default)?)?;
}
@@ -185,7 +187,7 @@ pub(crate) fn get_jit_args<'a>(
// fill in keyword only defaults
if let Some(kw_only_defaults) = kwdefaults {
for (i, name) in arg_names.kwonlyargs.iter().enumerate() {
let arg_idx = i + func.code.arg_count;
let arg_idx = i + func.code.arg_count as usize;
if !jit_args.is_set(arg_idx) {
let default = kw_only_defaults
.get_item(&**name, vm)
vm/src/builtins/traceback.rs
+1
View File
@@ -67,6 +67,7 @@ pub fn init(context: &Context) {
PyTraceback::extend_class(context, context.types.traceback_type);
}
#[cfg(feature = "serde")]
impl serde::Serialize for PyTraceback {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeStruct;
vm/src/exceptions.rs
+7 -5
View File
@@ -19,7 +19,6 @@ use itertools::Itertools;
use std::{
collections::HashSet,
io::{self, BufRead, BufReader},
ops::Deref,
};
impl std::fmt::Debug for PyBaseException {
@@ -958,22 +957,26 @@ fn system_exit_code(exc: PyBaseExceptionRef) -> Option<PyObjectRef> {
})
}
#[cfg(feature = "serde")]
pub struct SerializeException<'vm, 's> {
vm: &'vm VirtualMachine,
exc: &'s PyBaseExceptionRef,
}
#[cfg(feature = "serde")]
impl<'vm, 's> SerializeException<'vm, 's> {
pub fn new(vm: &'vm VirtualMachine, exc: &'s PyBaseExceptionRef) -> Self {
SerializeException { vm, exc }
}
}
#[cfg(feature = "serde")]
pub struct SerializeExceptionOwned<'vm> {
vm: &'vm VirtualMachine,
exc: PyBaseExceptionRef,
}
#[cfg(feature = "serde")]
impl serde::Serialize for SerializeExceptionOwned<'_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
let Self { vm, exc } = self;
@@ -981,12 +984,13 @@ impl serde::Serialize for SerializeExceptionOwned<'_> {
}
}
#[cfg(feature = "serde")]
impl serde::Serialize for SerializeException<'_, '_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::*;
let mut struc = s.serialize_struct("PyBaseException", 7)?;
struc.serialize_field("exc_type", self.exc.class().name().deref())?;
struc.serialize_field("exc_type", &*self.exc.class().name())?;
let tbs = {
struct Tracebacks(PyTracebackRef);
impl serde::Serialize for Tracebacks {
@@ -1111,8 +1115,6 @@ pub(super) mod types {
PyObjectRef, PyRef, PyResult, VirtualMachine,
};
use crossbeam_utils::atomic::AtomicCell;
#[cfg_attr(target_arch = "wasm32", allow(unused_imports))]
use std::ops::Deref;
// This module is designed to be used as `use builtins::*;`.
// Do not add any pub symbols not included in builtins module.
@@ -1332,7 +1334,7 @@ pub(super) mod types {
// We need this method, because of how `CPython` copies `init`
// from `BaseException` in `SimpleExtendsException` macro.
// See: `BaseException_new`
if cls.name().deref() == vm.ctx.exceptions.os_error.name().deref() {
if *cls.name() == *vm.ctx.exceptions.os_error.name() {
match os_error_optional_new(args.args.to_vec(), vm) {
Some(error) => error.to_pyresult(vm),
None => PyBaseException::slot_new(cls, args, vm),
vm/src/frame.rs
+14 -7
View File
@@ -513,10 +513,17 @@ impl ExecutingFrame<'_> {
Ok(None)
}
bytecode::Instruction::ImportName { idx } => {
self.import(vm, Some(self.code.names[idx.get(arg) as usize].to_owned()))
self.import(vm, Some(self.code.names[idx.get(arg) as usize].to_owned()))?;
Ok(None)
}
bytecode::Instruction::ImportNameless => {
self.import(vm, None)?;
Ok(None)
}
bytecode::Instruction::ImportStar => {
self.import_star(vm)?;
Ok(None)
}
bytecode::Instruction::ImportNameless => self.import(vm, None),
bytecode::Instruction::ImportStar => self.import_star(vm),
bytecode::Instruction::ImportFrom { idx } => {
let obj = self.import_from(vm, idx.get(arg))?;
self.push_value(obj);
@@ -1092,7 +1099,7 @@ impl ExecutingFrame<'_> {
}
#[cfg_attr(feature = "flame-it", flame("Frame"))]
fn import(&mut self, vm: &VirtualMachine, module: Option<PyStrRef>) -> FrameResult {
fn import(&mut self, vm: &VirtualMachine, module: Option<PyStrRef>) -> PyResult<()> {
let module = module.unwrap_or_else(|| vm.ctx.empty_str.clone());
let from_list = <Option<PyTupleTyped<PyStrRef>>>::try_from_object(vm, self.pop_value())?;
let level = usize::try_from_object(vm, self.pop_value())?;
@@ -1100,7 +1107,7 @@ impl ExecutingFrame<'_> {
let module = vm.import(module, from_list, level)?;
self.push_value(module);
Ok(None)
Ok(())
}
#[cfg_attr(feature = "flame-it", flame("Frame"))]
@@ -1127,7 +1134,7 @@ impl ExecutingFrame<'_> {
}
#[cfg_attr(feature = "flame-it", flame("Frame"))]
fn import_star(&mut self, vm: &VirtualMachine) -> FrameResult {
fn import_star(&mut self, vm: &VirtualMachine) -> PyResult<()> {
let module = self.pop_value();
// Grab all the names from the module and put them in the context
@@ -1150,7 +1157,7 @@ impl ExecutingFrame<'_> {
}
}
}
Ok(None)
Ok(())
}
/// Unwind blocks.
vm/src/lib.rs
+1
View File
@@ -64,6 +64,7 @@ pub mod object;
pub mod prelude;
pub mod protocol;
pub mod py_io;
#[cfg(feature = "serde")]
pub mod py_serde;
pub mod readline;
pub mod recursion;
vm/src/macros.rs
+2 -2
View File
@@ -144,8 +144,8 @@ macro_rules! match_class {
};
(match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => {
match $obj.payload::<$class>() {
Some($binding) => $expr,
None => $crate::match_class!(match ($obj) { $($rest)* }),
::std::option::Option::Some($binding) => $expr,
::std::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }),
}
};
vm/src/stdlib/marshal.rs
+148 -319
View File
@@ -2,207 +2,90 @@ pub(crate) use decl::make_module;
#[pymodule(name = "marshal")]
mod decl {
use crate::builtins::code::{CodeObject, Literal, PyObjBag};
use crate::class::StaticType;
use crate::{
builtins::{
PyBaseExceptionRef, PyByteArray, PyBytes, PyCode, PyDict, PyFloat, PyFrozenSet, PyInt,
PyList, PySet, PyStr, PyTuple,
PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat,
PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple,
},
bytecode,
convert::ToPyObject,
function::{ArgBytesLike, OptionalArg},
object::AsObject,
protocol::PyBuffer,
PyObjectRef, PyResult, TryFromObject, VirtualMachine,
};
use num_bigint::{BigInt, Sign};
use num_bigint::BigInt;
use num_complex::Complex64;
use num_traits::Zero;
#[repr(u8)]
enum Type {
// Null = b'0',
None = b'N',
False = b'F',
True = b'T',
StopIter = b'S',
Ellipsis = b'.',
Int = b'i',
Float = b'g',
// Complex = b'y',
// Long = b'l', // i32
Bytes = b's', // = TYPE_STRING
// Interned = b't',
// Ref = b'r',
Tuple = b'(',
List = b'[',
Dict = b'{',
Code = b'c',
Unicode = b'u',
// Unknown = b'?',
Set = b'<',
FrozenSet = b'>',
Ascii = b'a',
// AsciiInterned = b'A',
// SmallTuple = b')',
// ShortAscii = b'z',
// ShortAsciiInterned = b'Z',
}
// const FLAG_REF: u8 = b'\x80';
impl TryFrom<u8> for Type {
type Error = u8;
fn try_from(value: u8) -> Result<Self, u8> {
use Type::*;
Ok(match value {
// b'0' => Null,
b'N' => None,
b'F' => False,
b'T' => True,
b'S' => StopIter,
b'.' => Ellipsis,
b'i' => Int,
b'g' => Float,
// b'y' => Complex,
// b'l' => Long,
b's' => Bytes,
// b't' => Interned,
// b'r' => Ref,
b'(' => Tuple,
b'[' => List,
b'{' => Dict,
b'c' => Code,
b'u' => Unicode,
// b'?' => Unknown,
b'<' => Set,
b'>' => FrozenSet,
b'a' => Ascii,
// b'A' => AsciiInterned,
// b')' => SmallTuple,
// b'z' => ShortAscii,
// b'Z' => ShortAsciiInterned,
c => return Err(c),
})
}
}
use rustpython_compiler_core::marshal;
#[pyattr(name = "version")]
const VERSION: u32 = 4;
use marshal::FORMAT_VERSION;
fn too_short_error(vm: &VirtualMachine) -> PyBaseExceptionRef {
vm.new_exception_msg(
vm.ctx.exceptions.eof_error.to_owned(),
"marshal data too short".to_owned(),
)
}
pub struct DumpError;
/// Dumps a sequence of objects into binary vector.
fn dump_seq(
buf: &mut Vec<u8>,
iter: std::slice::Iter<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<()> {
write_size(buf, iter.len(), vm)?;
// For each element, dump into binary, then add its length and value.
for element in iter {
dump_obj(buf, element.clone(), vm)?;
}
Ok(())
}
/// Dumping helper function to turn a value into bytes.
fn dump_obj(buf: &mut Vec<u8>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
if vm.is_none(&value) {
buf.push(Type::None as u8);
} else if value.is(vm.ctx.exceptions.stop_iteration) {
buf.push(Type::StopIter as u8);
} else if value.is(&vm.ctx.ellipsis) {
buf.push(Type::Ellipsis as u8);
} else {
match_class!(match value {
pyint @ PyInt => {
if pyint.class().is(vm.ctx.types.bool_type) {
let typ = if pyint.as_bigint().is_zero() {
Type::False
} else {
Type::True
};
buf.push(typ as u8);
impl marshal::Dumpable for PyObjectRef {
type Error = DumpError;
type Constant = Literal;
fn with_dump<R>(
&self,
f: impl FnOnce(marshal::DumpableValue<'_, Self>) -> R,
) -> Result<R, Self::Error> {
use marshal::DumpableValue::*;
if self.is(PyStopIteration::static_type()) {
return Ok(f(StopIter));
}
let ret = match_class!(match self {
PyNone => f(None),
PyEllipsis => f(Ellipsis),
ref pyint @ PyInt => {
if self.class().is(PyBool::static_type()) {
f(Boolean(!pyint.as_bigint().is_zero()))
} else {
buf.push(Type::Int as u8);
let (sign, int_bytes) = pyint.as_bigint().to_bytes_le();
let mut len = int_bytes.len() as i32;
if sign == Sign::Minus {
len = -len;
}
buf.extend(len.to_le_bytes());
buf.extend(int_bytes);
f(Integer(pyint.as_bigint()))
}
}
pyfloat @ PyFloat => {
buf.push(Type::Float as u8);
buf.extend(pyfloat.to_f64().to_le_bytes());
ref pyfloat @ PyFloat => {
f(Float(pyfloat.to_f64()))
}
pystr @ PyStr => {
buf.push(if pystr.is_ascii() {
Type::Ascii
} else {
Type::Unicode
} as u8);
write_size(buf, pystr.as_str().len(), vm)?;
buf.extend(pystr.as_str().as_bytes());
ref pycomplex @ PyComplex => {
f(Complex(pycomplex.to_complex64()))
}
pylist @ PyList => {
buf.push(Type::List as u8);
let pylist_items = pylist.borrow_vec();
dump_seq(buf, pylist_items.iter(), vm)?;
ref pystr @ PyStr => {
f(Str(pystr.as_str()))
}
pyset @ PySet => {
buf.push(Type::Set as u8);
ref pylist @ PyList => {
f(List(&pylist.borrow_vec()))
}
ref pyset @ PySet => {
let elements = pyset.elements();
dump_seq(buf, elements.iter(), vm)?;
f(Set(&elements))
}
pyfrozen @ PyFrozenSet => {
buf.push(Type::FrozenSet as u8);
ref pyfrozen @ PyFrozenSet => {
let elements = pyfrozen.elements();
dump_seq(buf, elements.iter(), vm)?;
f(Frozenset(&elements))
}
pytuple @ PyTuple => {
buf.push(Type::Tuple as u8);
dump_seq(buf, pytuple.iter(), vm)?;
ref pytuple @ PyTuple => {
f(Tuple(pytuple.as_slice()))
}
pydict @ PyDict => {
buf.push(Type::Dict as u8);
write_size(buf, pydict.len(), vm)?;
for (key, value) in pydict {
dump_obj(buf, key, vm)?;
dump_obj(buf, value, vm)?;
}
ref pydict @ PyDict => {
let entries = pydict.into_iter().collect::<Vec<_>>();
f(Dict(&entries))
}
bytes @ PyBytes => {
buf.push(Type::Bytes as u8);
let data = bytes.as_bytes();
write_size(buf, data.len(), vm)?;
buf.extend(data);
ref bytes @ PyBytes => {
f(Bytes(bytes.as_bytes()))
}
bytes @ PyByteArray => {
buf.push(Type::Bytes as u8);
let data = bytes.borrow_buf();
write_size(buf, data.len(), vm)?;
buf.extend(&*data);
ref bytes @ PyByteArray => {
f(Bytes(&bytes.borrow_buf()))
}
co @ PyCode => {
buf.push(Type::Code as u8);
let bytes = co.code.map_clone_bag(&bytecode::BasicBag).to_bytes();
write_size(buf, bytes.len(), vm)?;
buf.extend(bytes);
ref co @ PyCode => {
f(Code(co))
}
_ => {
return Err(vm.new_not_implemented_error(
"TODO: not implemented yet or marshal unsupported type".to_owned(),
));
}
})
_ => return Err(DumpError),
});
Ok(ret)
}
Ok(())
}
#[pyfunction]
@@ -211,8 +94,16 @@ mod decl {
_version: OptionalArg<i32>,
vm: &VirtualMachine,
) -> PyResult<PyBytes> {
use marshal::Dumpable;
let mut buf = Vec::new();
dump_obj(&mut buf, value, vm)?;
value
.with_dump(|val| marshal::serialize_value(&mut buf, val))
.unwrap_or_else(Err)
.map_err(|DumpError| {
vm.new_not_implemented_error(
"TODO: not implemented yet or marshal unsupported type".to_owned(),
)
})?;
Ok(PyBytes::from(buf))
}
@@ -228,38 +119,83 @@ mod decl {
Ok(())
}
/// Safely convert usize to 4 le bytes
fn write_size(buf: &mut Vec<u8>, x: usize, vm: &VirtualMachine) -> PyResult<()> {
// For marshalling we want to convert lengths to bytes. To save space
// we limit the size to u32 to keep marshalling smaller.
let n = u32::try_from(x).map_err(|_| {
vm.new_value_error("Size exceeds 2^32 capacity for marshaling.".to_owned())
})?;
buf.extend(n.to_le_bytes());
Ok(())
}
#[derive(Copy, Clone)]
struct PyMarshalBag<'a>(&'a VirtualMachine);
/// Read the next 4 bytes of a slice, read as u32, pass as usize.
/// Returns the rest of buffer with the value.
fn read_size<'a>(buf: &'a [u8], vm: &VirtualMachine) -> PyResult<(usize, &'a [u8])> {
if buf.len() < 4 {
return Err(too_short_error(vm));
impl<'a> marshal::MarshalBag for PyMarshalBag<'a> {
type Value = PyObjectRef;
fn make_bool(&self, value: bool) -> Self::Value {
self.0.ctx.new_bool(value).into()
}
let (u32_bytes, rest) = buf.split_at(4);
let length = u32::from_le_bytes(u32_bytes.try_into().unwrap());
Ok((length as usize, rest))
}
/// Reads a list (or tuple) from a buffer.
fn load_seq<'b>(buf: &'b [u8], vm: &VirtualMachine) -> PyResult<(Vec<PyObjectRef>, &'b [u8])> {
let (len, mut buf) = read_size(buf, vm)?;
let mut elements: Vec<PyObjectRef> = Vec::new();
for _ in 0..len {
let (element, rest) = load_obj(buf, vm)?;
buf = rest;
elements.push(element);
fn make_none(&self) -> Self::Value {
self.0.ctx.none()
}
fn make_ellipsis(&self) -> Self::Value {
self.0.ctx.ellipsis()
}
fn make_float(&self, value: f64) -> Self::Value {
self.0.ctx.new_float(value).into()
}
fn make_complex(&self, value: Complex64) -> Self::Value {
self.0.ctx.new_complex(value).into()
}
fn make_str(&self, value: &str) -> Self::Value {
self.0.ctx.new_str(value).into()
}
fn make_bytes(&self, value: &[u8]) -> Self::Value {
self.0.ctx.new_bytes(value.to_vec()).into()
}
fn make_int(&self, value: BigInt) -> Self::Value {
self.0.ctx.new_int(value).into()
}
fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value {
self.0.ctx.new_tuple(elements.collect()).into()
}
fn make_code(&self, code: CodeObject) -> Self::Value {
self.0.ctx.new_code(code).into()
}
fn make_stop_iter(&self) -> Result<Self::Value, marshal::MarshalError> {
Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into())
}
fn make_list(
&self,
it: impl Iterator<Item = Self::Value>,
) -> Result<Self::Value, marshal::MarshalError> {
Ok(self.0.ctx.new_list(it.collect()).into())
}
fn make_set(
&self,
it: impl Iterator<Item = Self::Value>,
) -> Result<Self::Value, marshal::MarshalError> {
let vm = self.0;
let set = PySet::new_ref(&vm.ctx);
for elem in it {
set.add(elem, vm).unwrap()
}
Ok(set.into())
}
fn make_frozenset(
&self,
it: impl Iterator<Item = Self::Value>,
) -> Result<Self::Value, marshal::MarshalError> {
let vm = self.0;
Ok(PyFrozenSet::from_iter(vm, it).unwrap().to_pyobject(vm))
}
fn make_dict(
&self,
it: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value, marshal::MarshalError> {
let vm = self.0;
let dict = vm.ctx.new_dict();
for (k, v) in it {
dict.set_item(&*k, v, vm).unwrap()
}
Ok(dict.into())
}
type ConstantBag = PyObjBag<'a>;
fn constant_bag(self) -> Self::ConstantBag {
PyObjBag(&self.0.ctx)
}
Ok((elements, buf))
}
#[pyfunction]
@@ -267,128 +203,21 @@ mod decl {
let buf = pybuffer.as_contiguous().ok_or_else(|| {
vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous".to_owned())
})?;
let (obj, _) = load_obj(&buf, vm)?;
Ok(obj)
}
fn load_obj<'b>(buf: &'b [u8], vm: &VirtualMachine) -> PyResult<(PyObjectRef, &'b [u8])> {
let (type_indicator, buf) = buf.split_first().ok_or_else(|| too_short_error(vm))?;
let typ = Type::try_from(*type_indicator)
.map_err(|_| vm.new_value_error("bad marshal data (unknown type code)".to_owned()))?;
let (obj, buf) = match typ {
Type::True => (true.to_pyobject(vm), buf),
Type::False => (false.to_pyobject(vm), buf),
Type::None => (vm.ctx.none(), buf),
Type::StopIter => (
vm.ctx.exceptions.stop_iteration.to_owned().to_pyobject(vm),
buf,
marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e {
marshal::MarshalError::Eof => vm.new_exception_msg(
vm.ctx.exceptions.eof_error.to_owned(),
"marshal data too short".to_owned(),
),
Type::Ellipsis => (vm.ctx.ellipsis(), buf),
Type::Int => {
if buf.len() < 4 {
return Err(too_short_error(vm));
}
let (len_bytes, buf) = buf.split_at(4);
let len = i32::from_le_bytes(len_bytes.try_into().unwrap());
let (sign, len) = if len < 0 {
(Sign::Minus, (-len) as usize)
} else {
(Sign::Plus, len as usize)
};
if buf.len() < len {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(len);
let int = BigInt::from_bytes_le(sign, bytes);
(int.to_pyobject(vm), buf)
marshal::MarshalError::InvalidBytecode => {
vm.new_value_error("Couldn't deserialize python bytecode".to_owned())
}
Type::Float => {
if buf.len() < 8 {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(8);
let number = f64::from_le_bytes(bytes.try_into().unwrap());
(vm.ctx.new_float(number).into(), buf)
marshal::MarshalError::InvalidUtf8 => {
vm.new_value_error("invalid utf8 in marshalled string".to_owned())
}
Type::Ascii => {
let (len, buf) = read_size(buf, vm)?;
if buf.len() < len {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(len);
let s = String::from_utf8(bytes.to_vec())
.map_err(|_| vm.new_value_error("invalid utf8 data".to_owned()))?;
(s.to_pyobject(vm), buf)
marshal::MarshalError::BadType => {
vm.new_value_error("bad marshal data (unknown type code)".to_owned())
}
Type::Unicode => {
let (len, buf) = read_size(buf, vm)?;
if buf.len() < len {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(len);
let s = String::from_utf8(bytes.to_vec())
.map_err(|_| vm.new_value_error("invalid utf8 data".to_owned()))?;
(s.to_pyobject(vm), buf)
}
Type::List => {
let (elements, buf) = load_seq(buf, vm)?;
(vm.ctx.new_list(elements).into(), buf)
}
Type::Set => {
let (elements, buf) = load_seq(buf, vm)?;
let set = PySet::new_ref(&vm.ctx);
for element in elements {
set.add(element, vm)?;
}
(set.to_pyobject(vm), buf)
}
Type::FrozenSet => {
let (elements, buf) = load_seq(buf, vm)?;
let set = PyFrozenSet::from_iter(vm, elements.into_iter())?;
(set.to_pyobject(vm), buf)
}
Type::Tuple => {
let (elements, buf) = load_seq(buf, vm)?;
(vm.ctx.new_tuple(elements).into(), buf)
}
Type::Dict => {
let (len, mut buf) = read_size(buf, vm)?;
let dict = vm.ctx.new_dict();
for _ in 0..len {
let (key, rest) = load_obj(buf, vm)?;
let (value, rest) = load_obj(rest, vm)?;
buf = rest;
dict.set_item(key.as_object(), value, vm)?;
}
(dict.into(), buf)
}
Type::Bytes => {
// Following CPython, after marshaling, byte arrays are converted into bytes.
let (len, buf) = read_size(buf, vm)?;
if buf.len() < len {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(len);
(vm.ctx.new_bytes(bytes.to_vec()).into(), buf)
}
Type::Code => {
// If prefix is not identifiable, assume CodeObject, error out if it doesn't match.
let (len, buf) = read_size(buf, vm)?;
if buf.len() < len {
return Err(too_short_error(vm));
}
let (bytes, buf) = buf.split_at(len);
let code = bytecode::CodeObject::from_bytes(bytes).map_err(|e| match e {
bytecode::CodeDeserializeError::Eof => vm.new_exception_msg(
vm.ctx.exceptions.eof_error.to_owned(),
"End of file while deserializing bytecode".to_owned(),
),
_ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()),
})?;
(vm.ctx.new_code(code).into(), buf)
}
};
Ok((obj, buf))
})
}
#[pyfunction]
wasm/lib/Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ rustpython-parser = { path = "../../compiler/parser" }
rustpython-pylib = { path = "../../pylib", default-features = false, optional = true }
rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
# make sure no threading! otherwise wasm build will fail
rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] }
rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings", "serde"] }
serde = { workspace = true }