From 0f24d662344ded29a3025f21b45fc6dadd20e129 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 Mar 2023 18:55:46 +0900 Subject: [PATCH 01/20] pyclass macro to recognize Py/PyRef pattern --- derive-impl/src/pyclass.rs | 138 ++++++++++++++++++++++++++++++------- vm/src/builtins/code.rs | 2 +- vm/src/builtins/frame.rs | 4 +- vm/src/class.rs | 14 +--- vm/src/stdlib/io.rs | 8 +-- 5 files changed, 120 insertions(+), 46 deletions(-) diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index 92fd9e5bc..cd5bbc9a4 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -105,39 +105,114 @@ pub(crate) fn impl_pyimpl(attr: AttributeArgs, item: Item) -> Result { extract_items_into_context(&mut context, imp.items.iter_mut()); - let ty = &imp.self_ty; + let (impl_ty, payload_guess) = match imp.self_ty.as_ref() { + syn::Type::Path(syn::TypePath { + path: syn::Path { segments, .. }, + .. + }) if segments.len() == 1 => { + let segment = &segments[0]; + let payload_ty = if segment.ident == "Py" || segment.ident == "PyRef" { + match &segment.arguments { + syn::PathArguments::AngleBracketed( + syn::AngleBracketedGenericArguments { args, .. }, + ) if args.len() == 1 => { + let arg = &args[0]; + match arg { + syn::GenericArgument::Type(syn::Type::Path( + syn::TypePath { + path: syn::Path { segments, .. }, + .. + }, + )) if segments.len() == 1 => segments[0].ident.clone(), + _ => { + return Err(syn::Error::new_spanned( + segment, + "Py{Ref} is expected but Py{Ref} is found", + )) + } + } + } + _ => { + return Err(syn::Error::new_spanned( + segment, + "Py{Ref} is expected but Py{Ref}? is found", + )) + } + } + } else { + if !matches!(segment.arguments, syn::PathArguments::None) { + return Err(syn::Error::new_spanned( + segment, + "PyImpl can only be implemented for Py{Ref} or T", + )); + } + segment.ident.clone() + }; + (segment.ident.clone(), payload_ty) + } + _ => { + return Err(syn::Error::new_spanned( + imp.self_ty, + "PyImpl can only be implemented for Py{Ref} or T", + )) + } + }; + let ExtractedImplAttrs { + payload: attr_payload, with_impl, flags, with_slots, - } = extract_impl_attrs(attr, &Ident::new("e!(ty).to_string(), ty.span()))?; - + } = extract_impl_attrs(attr, &impl_ty)?; + let payload_ty = attr_payload.unwrap_or(payload_guess); let getset_impl = &context.getset_items; let member_impl = &context.member_items; let extend_impl = context.impl_extend_items.validate()?; let slots_impl = context.extend_slots_items.validate()?; let class_extensions = &context.class_extensions; - quote! { - #imp - impl ::rustpython_vm::class::PyClassImpl for #ty { - const TP_FLAGS: ::rustpython_vm::types::PyTypeFlags = #flags; - fn impl_extend_class( + let extra_methods = iter_chain![ + parse_quote! { + fn __extend_py_class( ctx: &::rustpython_vm::Context, class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, ) { #getset_impl #member_impl #extend_impl - #with_impl #(#class_extensions)* } - - fn extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #with_slots + }, + parse_quote! { + fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { #slots_impl } + }, + ]; + imp.items.extend(extra_methods); + let is_main_impl = impl_ty == payload_ty; + if is_main_impl { + quote! { + #imp + impl ::rustpython_vm::class::PyClassImpl for #payload_ty { + const TP_FLAGS: ::rustpython_vm::types::PyTypeFlags = #flags; + + fn impl_extend_class( + ctx: &::rustpython_vm::Context, + class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ) { + #impl_ty::__extend_py_class(ctx, class); + #with_impl + } + + fn extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { + #impl_ty::__extend_slots(slots); + #with_slots + } + } } + } else { + imp.into_token_stream() } } Item::Trait(mut trai) => { @@ -1163,6 +1238,7 @@ impl MemberItemMeta { } struct ExtractedImplAttrs { + payload: Option, with_impl: TokenStream, with_slots: TokenStream, flags: TokenStream, @@ -1182,6 +1258,7 @@ fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result Result::impl_extend_class), - quote!(PyRef::::extend_slots), - ) - } else { - ( - quote!(::__extend_py_class), - quote!(::__extend_slots), - ) - }; + let (extend_class, extend_slots) = + if path.is_ident("PyRef") || path.is_ident("Py") { + // special handling for PyRef + ( + quote!(#path::::__extend_py_class), + quote!(#path::::__extend_slots), + ) + } else { + ( + quote!(::__extend_py_class), + quote!(::__extend_slots), + ) + }; let item_span = item.span().resolved_at(Span::call_site()); withs.push(quote_spanned! { path.span() => #extend_class(ctx, class); @@ -1227,11 +1305,23 @@ fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result { + if path.is_ident("payload") { + if let syn::Lit::Str(lit) = lit { + payload = Some(Ident::new(&lit.value(), lit.span())); + } else { + bail_span!(lit, "payload must be a string literal") + } + } else { + bail_span!(path, "Unknown pyimpl attribute") + } + } attr => bail_span!(attr, "Unknown pyimpl attribute"), } } Ok(ExtractedImplAttrs { + payload, with_impl: quote! { #(#withs)* }, diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 2d44c2fa3..d9d28a70a 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -440,5 +440,5 @@ impl ToPyObject for bytecode::CodeObject { } pub fn init(ctx: &Context) { - PyRef::::extend_class(ctx, ctx.types.code_type); + PyCode::extend_class(ctx, ctx.types.code_type); } diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index e1f6f5105..10f9a1071 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -13,7 +13,7 @@ use crate::{ use num_traits::Zero; pub fn init(context: &Context) { - FrameRef::extend_class(context, context.types.frame_type); + Frame::extend_class(context, context.types.frame_type); } #[pyclass(with(Constructor, PyRef, Representable))] @@ -35,7 +35,7 @@ impl Representable for Frame { } #[pyclass] -impl FrameRef { +impl PyRef { #[pymethod] fn clear(self) { // TODO diff --git a/vm/src/class.rs b/vm/src/class.rs index a99e6874c..e07c5ae39 100644 --- a/vm/src/class.rs +++ b/vm/src/class.rs @@ -3,7 +3,7 @@ use crate::{ builtins::{PyBaseObject, PyBoundMethod, PyType, PyTypeRef}, identifier, - object::{Py, PyObjectPayload, PyObjectRef, PyRef}, + object::{Py, PyObjectRef}, types::{hash_not_implemented, PyTypeFlags, PyTypeSlots}, vm::Context, }; @@ -63,18 +63,6 @@ pub trait PyClassDef { const UNHASHABLE: bool = false; } -impl PyClassDef for PyRef -where - T: PyObjectPayload + PyClassDef, -{ - const NAME: &'static str = T::NAME; - const MODULE_NAME: Option<&'static str> = T::MODULE_NAME; - const TP_NAME: &'static str = T::TP_NAME; - const DOC: Option<&'static str> = T::DOC; - const BASICSIZE: usize = T::BASICSIZE; - const UNHASHABLE: bool = T::UNHASHABLE; -} - pub trait PyClassImpl: PyClassDef { const TP_FLAGS: PyTypeFlags = PyTypeFlags::DEFAULT; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 1b02f86eb..2062698b9 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -3065,8 +3065,6 @@ mod _io { closed: AtomicCell, } - type StringIORef = PyRef; - #[derive(FromArgs)] struct StringIONewArgs { #[pyarg(positional, optional)] @@ -3135,7 +3133,7 @@ mod _io { } #[pyclass] - impl StringIORef { + impl PyRef { //write string to underlying vector #[pymethod] fn write(self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { @@ -3216,8 +3214,6 @@ mod _io { exports: AtomicCell, } - type BytesIORef = PyRef; - impl Constructor for BytesIO { type Args = OptionalArg>; @@ -3261,7 +3257,7 @@ mod _io { } #[pyclass] - impl BytesIORef { + impl PyRef { #[pymethod] fn write(self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { let mut buffer = self.try_resizable(vm)?; From f314328294d9be04a1fe0629292fa3ff65ddbc5e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 Mar 2023 00:20:54 +0900 Subject: [PATCH 02/20] lifetime bounded TryFromBorrwoedObject --- examples/call_between_rust_and_python.rs | 4 +-- stdlib/src/mmap.rs | 4 +-- stdlib/src/resource.rs | 4 +-- stdlib/src/unicodedata.rs | 4 +-- stdlib/src/zlib.rs | 4 +-- vm/src/builtins/bool.rs | 4 +-- vm/src/builtins/bytes.rs | 4 +-- vm/src/builtins/int.rs | 4 +-- vm/src/builtins/str.rs | 4 +-- vm/src/bytesinner.rs | 4 +-- vm/src/convert/try_from.rs | 31 ++++++++++++++++++------ vm/src/function/buffer.rs | 8 +++--- vm/src/function/mod.rs | 4 +-- vm/src/protocol/buffer.rs | 4 +-- vm/src/stdlib/time.rs | 4 +-- 15 files changed, 53 insertions(+), 38 deletions(-) diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index c139309e8..98fb434fa 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -86,8 +86,8 @@ python_person.name: {}", name: String, } - impl TryFromBorrowedObject for PythonPerson { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for PythonPerson { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let name = obj.get_attr("name", vm)?.try_into_value::(vm)?; Ok(PythonPerson { name }) } diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index 06cc37d31..b0a72562e 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -72,8 +72,8 @@ mod mmap { Copy = 3, } - impl TryFromBorrowedObject for AccessMode { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for AccessMode { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let i = u32::try_from_borrowed_object(vm, obj)?; Ok(match i { 0 => Self::Default, diff --git a/stdlib/src/resource.rs b/stdlib/src/resource.rs index 1d3513f47..51f2fd785 100644 --- a/stdlib/src/resource.rs +++ b/stdlib/src/resource.rs @@ -129,8 +129,8 @@ mod resource { } struct Limits(libc::rlimit); - impl TryFromBorrowedObject for Limits { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for Limits { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let seq: Vec = obj.try_to_value(vm)?; match *seq { [cur, max] => Ok(Self(libc::rlimit { diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs index c8332740b..be7859843 100644 --- a/stdlib/src/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -41,8 +41,8 @@ enum NormalizeForm { Nfkd, } -impl TryFromBorrowedObject for NormalizeForm { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_value_with( |form: &PyStr| { Ok(match form.as_str() { diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index f7df60185..fd3efa28b 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -581,8 +581,8 @@ mod zlib { } } - impl TryFromBorrowedObject for Level { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for Level { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let int: i32 = obj.try_index(vm)?.try_to_primitive(vm)?; Ok(Self::new(int)) } diff --git a/vm/src/builtins/bool.rs b/vm/src/builtins/bool.rs index 4da54113f..e8d506827 100644 --- a/vm/src/builtins/bool.rs +++ b/vm/src/builtins/bool.rs @@ -19,8 +19,8 @@ impl ToPyObject for bool { } } -impl TryFromBorrowedObject for bool { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for bool { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { if obj.fast_isinstance(vm.ctx.types.int_type) { Ok(get_value(obj)) } else { diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 1d1cdfabb..f32f78422 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -742,8 +742,8 @@ impl IterNext for PyBytesIterator { } } -impl TryFromBorrowedObject for PyBytes { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for PyBytes { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { PyBytesInner::try_from_borrowed_object(vm, obj).map(|x| x.into()) } } diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index 9193bb793..46b81de56 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -72,8 +72,8 @@ impl_into_pyobject_int!(isize i8 i16 i32 i64 i128 usize u8 u16 u32 u64 u128 BigI macro_rules! impl_try_from_object_int { ($(($t:ty, $to_prim:ident),)*) => {$( - impl TryFromBorrowedObject for $t { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for $t { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_value_with(|int: &PyInt| { int.try_to_primitive(vm) }, vm) diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 8b6d61c37..32b86f0e6 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -42,8 +42,8 @@ use unic_ucd_category::GeneralCategory; use unic_ucd_ident::{is_xid_continue, is_xid_start}; use unicode_casing::CharExt; -impl TryFromBorrowedObject for String { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for String { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_value_with(|pystr: &PyStr| Ok(pystr.as_str().to_owned()), vm) } } diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index f2a0a7f35..4f07058a5 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -30,8 +30,8 @@ impl From> for PyBytesInner { } } -impl TryFromBorrowedObject for PyBytesInner { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for PyBytesInner { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { bytes_from_object(vm, obj).map(Self::from) } } diff --git a/vm/src/convert/try_from.rs b/vm/src/convert/try_from.rs index 42ab0d320..c1f51ebb5 100644 --- a/vm/src/convert/try_from.rs +++ b/vm/src/convert/try_from.rs @@ -1,7 +1,7 @@ use crate::{ builtins::PyFloat, object::{AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult}, - vm::VirtualMachine, + Py, VirtualMachine, }; use num_traits::ToPrimitive; @@ -15,7 +15,7 @@ pub trait TryFromObject: Sized { } /// Rust-side only version of TryFromObject to reduce unnecessary Rc::clone -impl TryFromObject for T { +impl TryFromBorrowedObject<'a>> TryFromObject for T { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { TryFromBorrowedObject::try_from_borrowed_object(vm, &obj) } @@ -31,9 +31,9 @@ impl PyObjectRef { } impl PyObject { - pub fn try_to_value(&self, vm: &VirtualMachine) -> PyResult + pub fn try_to_value<'a, T: 'a>(&'a self, vm: &VirtualMachine) -> PyResult where - T: TryFromBorrowedObject, + T: TryFromBorrowedObject<'a>, { T::try_from_borrowed_object(vm, self) } @@ -55,9 +55,12 @@ impl PyObject { } /// Lower-cost variation of `TryFromObject` -pub trait TryFromBorrowedObject: Sized { +pub trait TryFromBorrowedObject<'a>: Sized +where + Self: 'a, +{ /// Attempt to convert a Python object to a value of this type. - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult; + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult; } impl TryFromObject for PyRef @@ -93,12 +96,24 @@ impl TryFromObject for Option { } } -impl TryFromBorrowedObject for Vec { - fn try_from_borrowed_object(vm: &VirtualMachine, value: &PyObject) -> PyResult { +impl<'a, T: 'a + TryFromObject> TryFromBorrowedObject<'a> for Vec { + fn try_from_borrowed_object(vm: &VirtualMachine, value: &'a PyObject) -> PyResult { vm.extract_elements_with(value, |obj| T::try_from_object(vm, obj)) } } +impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { + let class = T::class(vm); + if obj.fast_isinstance(class) { + obj.downcast_ref() + .ok_or_else(|| vm.new_downcast_runtime_error(class, &obj)) + } else { + Err(vm.new_downcast_type_error(class, &obj)) + } + } +} + impl TryFromObject for std::time::Duration { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { use std::time::Duration; diff --git a/vm/src/function/buffer.rs b/vm/src/function/buffer.rs index 81e5e104c..2938e915e 100644 --- a/vm/src/function/buffer.rs +++ b/vm/src/function/buffer.rs @@ -65,8 +65,8 @@ impl From for PyBuffer { } } -impl TryFromBorrowedObject for ArgBytesLike { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; if buffer.desc.is_contiguous() { Ok(Self(buffer)) @@ -107,8 +107,8 @@ impl From for PyBuffer { } } -impl TryFromBorrowedObject for ArgMemoryBuffer { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for ArgMemoryBuffer { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; if !buffer.desc.is_contiguous() { Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())) diff --git a/vm/src/function/mod.rs b/vm/src/function/mod.rs index 5b4d2bdd7..b6199ab1e 100644 --- a/vm/src/function/mod.rs +++ b/vm/src/function/mod.rs @@ -28,8 +28,8 @@ pub enum ArgByteOrder { Little, } -impl TryFromBorrowedObject for ArgByteOrder { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for ArgByteOrder { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_value_with( |s: &PyStr| match s.as_str() { "big" => Ok(Self::Big), diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index 9ba0c0160..1450c837c 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -136,8 +136,8 @@ impl PyBuffer { } } -impl TryFromBorrowedObject for PyBuffer { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { +impl<'a> TryFromBorrowedObject<'a> for PyBuffer { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let cls = obj.class(); let as_buffer = cls.mro_find_map(|cls| cls.slots.as_buffer); if let Some(f) = as_buffer { diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 474d51d75..793a03c6d 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -435,8 +435,8 @@ mod unix { #[pyattr] use libc::{CLOCK_PROF, CLOCK_UPTIME}; - impl TryFromBorrowedObject for ClockId { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + impl<'a> TryFromBorrowedObject<'a> for ClockId { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_to_value(vm).map(ClockId::from_raw) } } From 1fda3ba9691c887a3644528d8970cdaca71e1382 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 Mar 2023 19:37:25 +0900 Subject: [PATCH 03/20] impl PyNativeFuncInternal for &Py form --- vm/src/function/builtin.rs | 33 +++++++++++++++++++++- vm/src/function/getset.rs | 56 ++++++++++++++++++++++++++++++++++++-- vm/src/function/mod.rs | 3 +- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/vm/src/function/builtin.rs b/vm/src/function/builtin.rs index 3c9a797a5..3a1591cb4 100644 --- a/vm/src/function/builtin.rs +++ b/vm/src/function/builtin.rs @@ -1,6 +1,7 @@ use super::{FromArgs, FuncArgs}; use crate::{ - convert::ToPyResult, object::PyThreadingConstraint, PyPayload, PyRef, PyResult, VirtualMachine, + convert::ToPyResult, object::PyThreadingConstraint, Py, PyPayload, PyRef, PyResult, + VirtualMachine, }; use std::marker::PhantomData; @@ -59,6 +60,8 @@ use sealed::PyNativeFuncInternal; #[doc(hidden)] pub struct OwnedParam(PhantomData); #[doc(hidden)] +pub struct BorrowedParam(PhantomData); +#[doc(hidden)] pub struct RefParam(PhantomData); // This is the "magic" that allows rust functions of varying signatures to @@ -80,6 +83,20 @@ macro_rules! into_py_native_func_tuple { } } + impl PyNativeFuncInternal<(BorrowedParam, $(OwnedParam<$T>,)*), R, VirtualMachine> for F + where + F: Fn(&Py, $($T,)* &VirtualMachine) -> R + PyThreadingConstraint + 'static, + S: PyPayload, + $($T: FromArgs,)* + R: ToPyResult, + { + fn call_(&self, vm: &VirtualMachine, args: FuncArgs) -> PyResult { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)* vm).to_pyresult(vm) + } + } + impl PyNativeFuncInternal<(RefParam, $(OwnedParam<$T>,)*), R, VirtualMachine> for F where F: Fn(&S, $($T,)* &VirtualMachine) -> R + PyThreadingConstraint + 'static, @@ -107,6 +124,20 @@ macro_rules! into_py_native_func_tuple { } } + impl PyNativeFuncInternal<(BorrowedParam, $(OwnedParam<$T>,)*), R, ()> for F + where + F: Fn(&Py, $($T,)*) -> R + PyThreadingConstraint + 'static, + S: PyPayload, + $($T: FromArgs,)* + R: ToPyResult, + { + fn call_(&self, vm: &VirtualMachine, args: FuncArgs) -> PyResult { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)*).to_pyresult(vm) + } + } + impl PyNativeFuncInternal<(RefParam, $(OwnedParam<$T>,)*), R, ()> for F where F: Fn(&S, $($T,)*) -> R + PyThreadingConstraint + 'static, diff --git a/vm/src/function/getset.rs b/vm/src/function/getset.rs index a7731f73c..827158e83 100644 --- a/vm/src/function/getset.rs +++ b/vm/src/function/getset.rs @@ -3,9 +3,9 @@ */ use crate::{ convert::ToPyResult, - function::{OwnedParam, RefParam}, + function::{BorrowedParam, OwnedParam, RefParam}, object::PyThreadingConstraint, - PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; #[derive(result_like::OptionLike, is_macro::Is, Debug)] @@ -74,6 +74,18 @@ where } } +impl IntoPyGetterFunc<(BorrowedParam, R, VirtualMachine)> for F +where + F: Fn(&Py, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::::try_from_object(vm, obj)?; + (self)(&zelf, vm).to_pyresult(vm) + } +} + impl IntoPyGetterFunc<(RefParam, R, VirtualMachine)> for F where F: Fn(&S, &VirtualMachine) -> R + 'static + Send + Sync, @@ -98,6 +110,18 @@ where } } +impl IntoPyGetterFunc<(BorrowedParam, R)> for F +where + F: Fn(&Py) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::::try_from_object(vm, obj)?; + (self)(&zelf).to_pyresult(vm) + } +} + impl IntoPyGetterFunc<(RefParam, R)> for F where F: Fn(&S) -> R + 'static + Send + Sync, @@ -149,6 +173,20 @@ where } } +impl IntoPySetterFunc<(BorrowedParam, V, R, VirtualMachine)> for F +where + F: Fn(&Py, V, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value, vm).into_noresult() + } +} + impl IntoPySetterFunc<(RefParam, V, R, VirtualMachine)> for F where F: Fn(&S, V, &VirtualMachine) -> R + 'static + Send + Sync, @@ -177,6 +215,20 @@ where } } +impl IntoPySetterFunc<(BorrowedParam, V, R)> for F +where + F: Fn(&Py, V) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value).into_noresult() + } +} + impl IntoPySetterFunc<(RefParam, V, R)> for F where F: Fn(&S, V) -> R + 'static + Send + Sync, diff --git a/vm/src/function/mod.rs b/vm/src/function/mod.rs index b6199ab1e..3a097acf8 100644 --- a/vm/src/function/mod.rs +++ b/vm/src/function/mod.rs @@ -13,7 +13,8 @@ pub use argument::{ }; pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike}; -pub use builtin::{IntoPyNativeFunc, OwnedParam, PyNativeFunc, RefParam}; +pub(self) use builtin::{BorrowedParam, OwnedParam, RefParam}; +pub use builtin::{IntoPyNativeFunc, PyNativeFunc}; pub use either::Either; pub use getset::PySetterValue; pub(super) use getset::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc}; From 174ef52df7e33ca36f0d60c8e7174d7bc42c8190 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 Mar 2023 00:21:03 +0900 Subject: [PATCH 04/20] \#[pyclass] impl Py for --- vm/src/builtins/type.rs | 129 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index f091262a6..5b9c1a117 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -355,6 +355,31 @@ impl PyType { attributes } + + // bound method for every type + pub(crate) fn __new__(zelf: PyRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (subtype, args): (PyRef, FuncArgs) = args.bind(vm)?; + if !subtype.fast_issubclass(&zelf) { + return Err(vm.new_type_error(format!( + "{zelf}.__new__({subtype}): {subtype} is not a subtype of {zelf}", + zelf = zelf.name(), + subtype = subtype.name(), + ))); + } + call_slot_new(zelf, subtype, args, vm) + } + + pub fn name(&self) -> BorrowedValue { + PyRwLockReadGuard::map(self.slots.name.read(), |slot_name| { + let name = slot_name.as_ref().unwrap(); + if self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + name.as_str() + } else { + name.rsplit('.').next().unwrap() + } + }) + .into() + } } impl Py { @@ -375,30 +400,10 @@ impl Py { } #[pyclass( - with(GetAttr, SetAttr, Callable, AsNumber, Representable), + with(Py, GetAttr, SetAttr, Callable, AsNumber, Representable), flags(BASETYPE) )] impl PyType { - // bound method for every type - pub(crate) fn __new__(zelf: PyRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let (subtype, args): (PyRef, FuncArgs) = args.bind(vm)?; - if !subtype.fast_issubclass(&zelf) { - return Err(vm.new_type_error(format!( - "{zelf}.__new__({subtype}): {subtype} is not a subtype of {zelf}", - zelf = zelf.name(), - subtype = subtype.name(), - ))); - } - call_slot_new(zelf, subtype, args, vm) - } - - #[pygetset(name = "__mro__")] - fn get_mro(zelf: PyRef) -> PyTuple { - let elements: Vec = - zelf.iter_mro().map(|x| x.as_object().to_owned()).collect(); - PyTuple::new_unchecked(elements.into_boxed_slice()) - } - #[pygetset(magic)] fn bases(&self, vm: &VirtualMachine) -> PyTupleRef { vm.ctx.new_tuple( @@ -419,48 +424,11 @@ impl PyType { self.slots.flags.bits() } - #[pymethod(magic)] - fn dir(zelf: PyRef, _vm: &VirtualMachine) -> PyList { - let attributes: Vec = zelf - .get_attributes() - .into_iter() - .map(|(k, _)| k.to_object()) - .collect(); - PyList::from(attributes) - } - - #[pymethod(magic)] - fn instancecheck(zelf: PyRef, obj: PyObjectRef) -> bool { - obj.fast_isinstance(&zelf) - } - - #[pymethod(magic)] - fn subclasscheck(zelf: PyRef, subclass: PyTypeRef) -> bool { - subclass.fast_issubclass(&zelf) - } - - #[pyclassmethod(magic)] - fn subclasshook(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - #[pygetset] fn __name__(&self) -> String { self.name().to_string() } - pub fn name(&self) -> BorrowedValue { - PyRwLockReadGuard::map(self.slots.name.read(), |slot_name| { - let name = slot_name.as_ref().unwrap(); - if self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - name.as_str() - } else { - name.rsplit('.').next().unwrap() - } - }) - .into() - } - #[pygetset(magic)] pub fn qualname(&self, vm: &VirtualMachine) -> PyObjectRef { self.attributes @@ -607,11 +575,6 @@ impl PyType { ) } - #[pymethod] - fn mro(zelf: PyRef) -> Vec { - zelf.iter_mro().map(|cls| cls.to_owned().into()).collect() - } - #[pymethod(magic)] pub fn ror(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { or_(other, zelf, vm) @@ -906,6 +869,46 @@ impl PyType { } } +#[pyclass] +impl Py { + #[pygetset(name = "__mro__")] + fn get_mro(&self) -> PyTuple { + let elements: Vec = + self.iter_mro().map(|x| x.as_object().to_owned()).collect(); + PyTuple::new_unchecked(elements.into_boxed_slice()) + } + + #[pymethod(magic)] + fn dir(&self) -> PyList { + let attributes: Vec = self + .get_attributes() + .into_iter() + .map(|(k, _)| k.to_object()) + .collect(); + PyList::from(attributes) + } + + #[pymethod(magic)] + fn instancecheck(&self, obj: PyObjectRef) -> bool { + obj.fast_isinstance(self) + } + + #[pymethod(magic)] + fn subclasscheck(&self, subclass: PyTypeRef) -> bool { + subclass.fast_issubclass(self) + } + + #[pyclassmethod(magic)] + fn subclasshook(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() + } + + #[pymethod] + fn mro(&self) -> Vec { + self.iter_mro().map(|cls| cls.to_owned().into()).collect() + } +} + const SIGNATURE_END_MARKER: &str = ")\n--\n\n"; fn get_signature(doc: &str) -> Option<&str> { doc.find(SIGNATURE_END_MARKER).map(|index| &doc[..=index]) From b052d646ee0e8af20375138117e5388670c1feaf Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 18:02:28 +0900 Subject: [PATCH 05/20] clean up PyCode --- vm/src/builtins/code.rs | 54 +++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index d9d28a70a..34ed065b6 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -10,7 +10,7 @@ use crate::{ convert::ToPyObject, function::{FuncArgs, OptionalArg}, types::Representable, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use num_traits::Zero; use std::{borrow::Borrow, fmt, ops::Deref}; @@ -216,9 +216,6 @@ impl PyPayload for PyCode { } } -#[pyclass(with(PyRef, Representable))] -impl PyCode {} - impl Representable for PyCode { #[inline] fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { @@ -233,35 +230,35 @@ impl Representable for PyCode { } } -#[pyclass] -impl PyRef { +#[pyclass(with(Py))] +impl PyCode { #[pyslot] fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("Cannot directly create code object".to_owned())) } #[pygetset] - fn co_posonlyargcount(self) -> usize { + fn co_posonlyargcount(&self) -> usize { self.code.posonlyarg_count as usize } #[pygetset] - fn co_argcount(self) -> usize { + fn co_argcount(&self) -> usize { self.code.arg_count as usize } #[pygetset] - fn co_stacksize(self) -> u32 { + fn co_stacksize(&self) -> u32 { self.code.max_stackdepth } #[pygetset] - pub fn co_filename(self) -> PyStrRef { + pub fn co_filename(&self) -> PyStrRef { self.code.source_path.to_owned() } #[pygetset] - pub fn co_cellvars(self, vm: &VirtualMachine) -> PyTupleRef { + pub fn co_cellvars(&self, vm: &VirtualMachine) -> PyTupleRef { let cellvars = self .code .cellvars @@ -273,33 +270,33 @@ impl PyRef { } #[pygetset] - fn co_nlocals(self) -> usize { + fn co_nlocals(&self) -> usize { self.varnames.len() } #[pygetset] - fn co_firstlineno(self) -> usize { + fn co_firstlineno(&self) -> usize { self.code.first_line_number as usize } #[pygetset] - fn co_kwonlyargcount(self) -> usize { + fn co_kwonlyargcount(&self) -> usize { self.code.kwonlyarg_count as usize } #[pygetset] - fn co_consts(self, vm: &VirtualMachine) -> PyTupleRef { + fn co_consts(&self, vm: &VirtualMachine) -> PyTupleRef { let consts = self.code.constants.iter().map(|x| x.0.clone()).collect(); vm.ctx.new_tuple(consts) } #[pygetset] - fn co_name(self) -> PyStrRef { + fn co_name(&self) -> PyStrRef { self.code.obj_name.to_owned() } #[pygetset] - fn co_names(self, vm: &VirtualMachine) -> PyTupleRef { + fn co_names(&self, vm: &VirtualMachine) -> PyTupleRef { let names = self .code .names @@ -311,18 +308,18 @@ impl PyRef { } #[pygetset] - fn co_flags(self) -> u16 { + fn co_flags(&self) -> u16 { self.code.flags.bits() } #[pygetset] - pub fn co_varnames(self, vm: &VirtualMachine) -> PyTupleRef { + pub fn co_varnames(&self, vm: &VirtualMachine) -> PyTupleRef { let varnames = self.code.varnames.iter().map(|s| s.to_object()).collect(); vm.ctx.new_tuple(varnames) } #[pygetset] - pub fn co_freevars(self, vm: &VirtualMachine) -> PyTupleRef { + pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef { let names = self .code .freevars @@ -334,7 +331,7 @@ impl PyRef { } #[pymethod] - pub fn replace(self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult { + pub fn replace(&self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult { let posonlyarg_count = match args.co_posonlyargcount { OptionalArg::Present(posonlyarg_count) => posonlyarg_count, OptionalArg::Missing => self.code.posonlyarg_count, @@ -421,6 +418,21 @@ impl PyRef { } } +#[pyclass] +impl Py { + #[pymethod(magic)] + fn repr(&self) -> String { + let code = &self.code; + format!( + "", + code.obj_name, + self.get_id(), + code.source_path.as_str(), + code.first_line_number + ) + } +} + impl fmt::Display for PyCode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) From 2cc83ed1fef04fd7deac28174a3a5ac2ca5a299b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 18:39:05 +0900 Subject: [PATCH 06/20] clean up Frame --- stdlib/src/faulthandler.rs | 6 +-- vm/src/builtins/frame.rs | 56 +++++++++++++-------------- vm/src/frame.rs | 78 ++++++++++++++++++++------------------ 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index 06c893a3a..927ecd94a 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -2,11 +2,9 @@ pub(crate) use decl::make_module; #[pymodule(name = "faulthandler")] mod decl { - use crate::vm::{ - frame::FrameRef, function::OptionalArg, stdlib::sys::PyStderr, VirtualMachine, - }; + use crate::vm::{frame::Frame, function::OptionalArg, stdlib::sys::PyStderr, VirtualMachine}; - fn dump_frame(frame: &FrameRef, vm: &VirtualMachine) { + fn dump_frame(frame: &Frame, vm: &VirtualMachine) { let stderr = PyStderr(vm); writeln!( stderr, diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index 10f9a1071..f08767af9 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -16,9 +16,6 @@ pub fn init(context: &Context) { Frame::extend_class(context, context.types.frame_type); } -#[pyclass(with(Constructor, PyRef, Representable))] -impl Frame {} - impl Unconstructible for Frame {} impl Representable for Frame { @@ -34,61 +31,46 @@ impl Representable for Frame { } } -#[pyclass] -impl PyRef { +#[pyclass(with(Constructor, Py))] +impl Frame { #[pymethod] - fn clear(self) { + fn clear(&self) { // TODO } #[pygetset] - fn f_globals(self) -> PyDictRef { + fn f_globals(&self) -> PyDictRef { self.globals.clone() } #[pygetset] - fn f_locals(self, vm: &VirtualMachine) -> PyResult { + fn f_locals(&self, vm: &VirtualMachine) -> PyResult { self.locals(vm).map(Into::into) } #[pygetset] - pub fn f_code(self) -> PyRef { + pub fn f_code(&self) -> PyRef { self.code.clone() } #[pygetset] - pub fn f_back(self, vm: &VirtualMachine) -> Option { - // TODO: actually store f_back inside Frame struct - - // get the frame in the frame stack that appears before this one. - // won't work if this frame isn't in the frame stack, hence the todo above - vm.frames - .borrow() - .iter() - .rev() - .skip_while(|p| !p.is(&self)) - .nth(1) - .cloned() - } - - #[pygetset] - fn f_lasti(self) -> u32 { + fn f_lasti(&self) -> u32 { self.lasti() } #[pygetset] - pub fn f_lineno(self) -> usize { + pub fn f_lineno(&self) -> usize { self.current_location().row() } #[pygetset] - fn f_trace(self) -> PyObjectRef { + fn f_trace(&self) -> PyObjectRef { let boxed = self.trace.lock(); boxed.clone() } #[pygetset(setter)] - fn set_f_trace(self, value: PySetterValue, vm: &VirtualMachine) { + fn set_f_trace(&self, value: PySetterValue, vm: &VirtualMachine) { let mut storage = self.trace.lock(); *storage = value.unwrap_or_none(vm); } @@ -126,3 +108,21 @@ impl PyRef { } } } + +#[pyclass] +impl Py { + #[pygetset] + pub fn f_back(&self, vm: &VirtualMachine) -> Option> { + // TODO: actually store f_back inside Frame struct + + // get the frame in the frame stack that appears before this one. + // won't work if this frame isn't in the frame stack, hence the todo above + vm.frames + .borrow() + .iter() + .rev() + .skip_while(|p| !p.is(self.as_object())) + .nth(1) + .cloned() + } +} diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 821a02b3c..d5c77c0e3 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -164,24 +164,20 @@ impl Frame { temporary_refs: PyMutex::new(vec![]), } } -} -impl FrameRef { - #[inline(always)] - fn with_exec(&self, f: impl FnOnce(ExecutingFrame) -> R) -> R { - let mut state = self.state.lock(); - let exec = ExecutingFrame { - code: &self.code, - fastlocals: &self.fastlocals, - cells_frees: &self.cells_frees, - locals: &self.locals, - globals: &self.globals, - builtins: &self.builtins, - lasti: &self.lasti, - object: self, - state: &mut state, - }; - f(exec) + pub fn current_location(&self) -> bytecode::Location { + self.code.locations[self.lasti() as usize - 1] + } + + pub fn lasti(&self) -> u32 { + #[cfg(feature = "threading")] + { + self.lasti.load(atomic::Ordering::Relaxed) + } + #[cfg(not(feature = "threading"))] + { + self.lasti.get() + } } pub fn locals(&self, vm: &VirtualMachine) -> PyResult { @@ -221,6 +217,25 @@ impl FrameRef { } Ok(locals.clone()) } +} + +impl Py { + #[inline(always)] + fn with_exec(&self, f: impl FnOnce(ExecutingFrame) -> R) -> R { + let mut state = self.state.lock(); + let exec = ExecutingFrame { + code: &self.code, + fastlocals: &self.fastlocals, + cells_frees: &self.cells_frees, + locals: &self.locals, + globals: &self.globals, + builtins: &self.builtins, + lasti: &self.lasti, + object: self, + state: &mut state, + }; + f(exec) + } // #[cfg_attr(feature = "flame-it", flame("Frame"))] pub fn run(&self, vm: &VirtualMachine) -> PyResult { @@ -250,34 +265,19 @@ impl FrameRef { self.with_exec(|mut exec| exec.gen_throw(vm, exc_type, exc_val, exc_tb)) } - pub fn current_location(&self) -> bytecode::Location { - self.code.locations[self.lasti() as usize - 1] - } - pub fn yield_from_target(&self) -> Option { self.with_exec(|exec| exec.yield_from_target().map(PyObject::to_owned)) } - pub fn lasti(&self) -> u32 { - #[cfg(feature = "threading")] - { - self.lasti.load(atomic::Ordering::Relaxed) - } - #[cfg(not(feature = "threading"))] - { - self.lasti.get() - } - } - pub fn is_internal_frame(&self) -> bool { - let code = self.clone().f_code(); + let code = self.f_code(); let filename = code.co_filename(); filename.as_str().contains("importlib") && filename.as_str().contains("_bootstrap") } pub fn next_external_frame(&self, vm: &VirtualMachine) -> Option { - self.clone().f_back(vm).map(|mut back| loop { + self.f_back(vm).map(|mut back| loop { back = if let Some(back) = back.to_owned().f_back(vm) { back } else { @@ -300,7 +300,7 @@ struct ExecutingFrame<'a> { locals: &'a ArgMapping, globals: &'a PyDictRef, builtins: &'a PyDictRef, - object: &'a FrameRef, + object: &'a Py, lasti: &'a Lasti, state: &'a mut FrameState, } @@ -376,8 +376,12 @@ impl ExecutingFrame<'_> { let loc = frame.code.locations[idx]; let next = exception.traceback(); - let new_traceback = - PyTraceback::new(next, frame.object.clone(), frame.lasti(), loc.row()); + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + frame.lasti(), + loc.row(), + ); vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row()); exception.set_traceback(Some(new_traceback.into_ref(vm))); From e95eafe02affe858c3d9f8f2479d189c4076d82a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 19:05:59 +0900 Subject: [PATCH 07/20] clean up io.{String,Bytes}IO --- .cspell.json | 1 + vm/src/stdlib/io.rs | 114 ++++++++++++++++++++------------------------ 2 files changed, 52 insertions(+), 63 deletions(-) diff --git a/.cspell.json b/.cspell.json index 276869671..3f60bb076 100644 --- a/.cspell.json +++ b/.cspell.json @@ -61,6 +61,7 @@ "rsplitn", "rustc", "rustfmt", + "seekfrom", "splitn", "subsec", "timsort", diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 2062698b9..630ad653a 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -3098,7 +3098,6 @@ mod _io { } } - #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor))] impl StringIO { fn buffer(&self, vm: &VirtualMachine) -> PyResult> { if !self.closed.load() { @@ -3107,7 +3106,10 @@ mod _io { Err(io_closed_error(vm)) } } + } + #[pyclass(flags(BASETYPE, HAS_DICT), with(Constructor))] + impl StringIO { #[pymethod] fn readable(&self) -> bool { true @@ -3130,34 +3132,28 @@ mod _io { fn close(&self) { self.closed.store(true); } - } - #[pyclass] - impl PyRef { - //write string to underlying vector + // write string to underlying vector #[pymethod] - fn write(self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { + fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { let bytes = data.as_str().as_bytes(); - - match self.buffer(vm)?.write(bytes) { - Some(value) => Ok(vm.ctx.new_int(value).into()), - None => Err(vm.new_type_error("Error Writing String".to_owned())), - } + self.buffer(vm)? + .write(bytes) + .ok_or_else(|| vm.new_type_error("Error Writing String".to_owned())) } - //return the entire contents of the underlying + // return the entire contents of the underlying #[pymethod] - fn getvalue(self, vm: &VirtualMachine) -> PyResult { - match String::from_utf8(self.buffer(vm)?.getvalue()) { - Ok(result) => Ok(vm.ctx.new_str(result).into()), - Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_owned())), - } + fn getvalue(&self, vm: &VirtualMachine) -> PyResult { + let bytes = self.buffer(vm)?.getvalue(); + String::from_utf8(bytes) + .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) } - //skip to the jth position + // skip to the jth position #[pymethod] fn seek( - self, + &self, offset: PyObjectRef, how: OptionalArg, vm: &VirtualMachine, @@ -3167,38 +3163,34 @@ mod _io { .map_err(|err| os_err(vm, err)) } - //Read k bytes from the object and return. - //If k is undefined || k == -1, then we read all bytes until the end of the file. - //This also increments the stream position by the value of k + // Read k bytes from the object and return. + // If k is undefined || k == -1, then we read all bytes until the end of the file. + // This also increments the stream position by the value of k #[pymethod] - fn read(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { - let data = match self.buffer(vm)?.read(size.to_usize()) { - Some(value) => value, - None => Vec::new(), - }; + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + let data = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); let value = String::from_utf8(data) .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned()))?; - Ok(vm.ctx.new_str(value).into()) + Ok(value) } #[pymethod] - fn tell(self, vm: &VirtualMachine) -> PyResult { + fn tell(&self, vm: &VirtualMachine) -> PyResult { Ok(self.buffer(vm)?.tell()) } #[pymethod] - fn readline(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { // TODO size should correspond to the number of characters, at the moments its the number of // bytes. - match String::from_utf8(self.buffer(vm)?.readline(size.to_usize(), vm)?) { - Ok(value) => Ok(value), - Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_owned())), - } + let input = self.buffer(vm)?.readline(size.to_usize(), vm)?; + String::from_utf8(input) + .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) } #[pymethod] - fn truncate(self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult { + fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult { let mut buffer = self.buffer(vm)?; let pos = pos.try_usize(vm)?; Ok(buffer.truncate(pos)) @@ -3232,7 +3224,6 @@ mod _io { } } - #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor))] impl BytesIO { fn buffer(&self, vm: &VirtualMachine) -> PyResult> { if !self.closed.load() { @@ -3241,7 +3232,10 @@ mod _io { Err(io_closed_error(vm)) } } + } + #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor))] + impl BytesIO { #[pymethod] fn readable(&self) -> bool { true @@ -3254,37 +3248,33 @@ mod _io { fn seekable(&self) -> bool { true } - } - #[pyclass] - impl PyRef { #[pymethod] - fn write(self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { + fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { let mut buffer = self.try_resizable(vm)?; - match data.with_ref(|b| buffer.write(b)) { - Some(value) => Ok(value), - None => Err(vm.new_type_error("Error Writing Bytes".to_owned())), - } + data.with_ref(|b| buffer.write(b)) + .ok_or_else(|| vm.new_type_error("Error Writing Bytes".to_owned())) } - //Retrieves the entire bytes object value from the underlying buffer + // Retrieves the entire bytes object value from the underlying buffer #[pymethod] - fn getvalue(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_bytes(self.buffer(vm)?.getvalue())) + fn getvalue(&self, vm: &VirtualMachine) -> PyResult { + let bytes = self.buffer(vm)?.getvalue(); + Ok(vm.ctx.new_bytes(bytes)) } - //Takes an integer k (bytes) and returns them from the underlying buffer - //If k is undefined || k == -1, then we read all bytes until the end of the file. - //This also increments the stream position by the value of k + // Takes an integer k (bytes) and returns them from the underlying buffer + // If k is undefined || k == -1, then we read all bytes until the end of the file. + // This also increments the stream position by the value of k #[pymethod] #[pymethod(name = "read1")] - fn read(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult> { + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult> { let buf = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); Ok(buf) } #[pymethod] - fn readinto(self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { + fn readinto(&self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { let mut buf = self.buffer(vm)?; let ret = buf .cursor @@ -3297,7 +3287,7 @@ mod _io { //skip to the jth position #[pymethod] fn seek( - self, + &self, offset: PyObjectRef, how: OptionalArg, vm: &VirtualMachine, @@ -3308,22 +3298,17 @@ mod _io { } #[pymethod] - fn seekable(self) -> bool { - true - } - - #[pymethod] - fn tell(self, vm: &VirtualMachine) -> PyResult { + fn tell(&self, vm: &VirtualMachine) -> PyResult { Ok(self.buffer(vm)?.tell()) } #[pymethod] - fn readline(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult> { + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult> { self.buffer(vm)?.readline(size.to_usize(), vm) } #[pymethod] - fn truncate(self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult { + fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult { if self.closed.load() { return Err(io_closed_error(vm)); } @@ -3333,17 +3318,20 @@ mod _io { } #[pygetset] - fn closed(self) -> bool { + fn closed(&self) -> bool { self.closed.load() } #[pymethod] - fn close(self, vm: &VirtualMachine) -> PyResult<()> { + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { drop(self.try_resizable(vm)?); self.closed.store(true); Ok(()) } + } + #[pyclass] + impl PyRef { #[pymethod] fn getbuffer(self, vm: &VirtualMachine) -> PyResult { let len = self.buffer.read().cursor.get_ref().len(); From 13b4fd9b9fba5b4451fdf5028367d4b27a2ab930 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 22:59:51 +0900 Subject: [PATCH 08/20] {PyRef->Py}::upgrade --- vm/src/object/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 879030208..1de24bdec 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -385,7 +385,7 @@ impl Drop for PyWeak { } } -impl PyRef { +impl Py { #[inline(always)] pub fn upgrade(&self) -> Option { PyWeak::upgrade(self) From 47d2e7b6581e416caa4d8ac4fc8ef43b2be40686 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 19:25:04 +0900 Subject: [PATCH 09/20] clean up PyArray --- stdlib/src/array.rs | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index b9481eaba..5becd160d 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -746,7 +746,7 @@ mod array { } #[pymethod] - fn append(zelf: PyRef, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn append(zelf: &Py, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { zelf.try_resizable(vm)?.push(x, vm) } @@ -762,12 +762,12 @@ mod array { } #[pymethod] - fn remove(zelf: PyRef, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn remove(zelf: &Py, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { zelf.try_resizable(vm)?.remove(x, vm) } #[pymethod] - fn extend(zelf: PyRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn extend(zelf: &Py, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let mut w = zelf.try_resizable(vm)?; if zelf.is(&obj) { w.imul(2, vm) @@ -828,7 +828,7 @@ mod array { } #[pymethod] - fn fromunicode(zelf: PyRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn fromunicode(zelf: &Py, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let utf8 = PyStrRef::try_from_object(vm, obj.clone()).map_err(|_| { vm.new_type_error(format!( "fromunicode() argument must be str, not {}", @@ -922,18 +922,13 @@ mod array { } #[pymethod] - fn insert( - zelf: PyRef, - i: isize, - x: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { + fn insert(zelf: &Py, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let mut w = zelf.try_resizable(vm)?; w.insert(i, x, vm) } #[pymethod] - fn pop(zelf: PyRef, i: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn pop(zelf: &Py, i: OptionalArg, vm: &VirtualMachine) -> PyResult { let mut w = zelf.try_resizable(vm)?; if w.len() == 0 { Err(vm.new_index_error("pop from empty array".to_owned())) @@ -982,7 +977,7 @@ mod array { } #[pymethod] - fn fromlist(zelf: PyRef, list: PyListRef, vm: &VirtualMachine) -> PyResult<()> { + fn fromlist(zelf: &Py, list: PyListRef, vm: &VirtualMachine) -> PyResult<()> { zelf.try_resizable(vm)?.fromlist(&list, vm) } @@ -1014,7 +1009,7 @@ mod array { } fn _setitem( - zelf: PyRef, + zelf: &Py, needle: &PyObject, value: PyObjectRef, vm: &VirtualMachine, @@ -1052,7 +1047,7 @@ mod array { #[pymethod(magic)] fn setitem( - zelf: PyRef, + zelf: &Py, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine, @@ -1120,7 +1115,7 @@ mod array { } #[pymethod(magic)] - fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { let class = zelf.class(); let class_name = class.name(); if zelf.read().typecode() == 'u' { @@ -1167,7 +1162,7 @@ mod array { #[pymethod(magic)] fn reduce_ex( - zelf: PyRef, + zelf: &Py, proto: usize, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, PyTupleRef, Option)> { @@ -1191,7 +1186,7 @@ mod array { #[pymethod(magic)] fn reduce( - zelf: PyRef, + zelf: &Py, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, PyTupleRef, Option)> { let array = zelf.read(); @@ -1325,7 +1320,7 @@ mod array { ass_subscript: atomic_func!(|mapping, needle, value, vm| { let zelf = PyArray::mapping_downcast(mapping); if let Some(value) = value { - PyArray::_setitem(zelf.to_owned(), needle, value, vm) + PyArray::_setitem(zelf, needle, value, vm) } else { zelf._delitem(needle, vm) } From 107a64003d4afe31c84f372353fe1120c7af4432 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Mar 2023 23:37:09 +0900 Subject: [PATCH 10/20] clean up sqlite a bit --- stdlib/src/sqlite.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 8982b80c0..cd8a95cd2 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -883,7 +883,8 @@ mod _sqlite { unsafe { cursor.row_factory.swap(zelf.row_factory.to_owned()) }; cursor } else { - Cursor::new(zelf.clone(), zelf.row_factory.to_owned(), vm).into_ref(vm) + let row_factory = zelf.row_factory.to_owned(); + Cursor::new(zelf, row_factory, vm).into_ref(vm) }; Ok(cursor) } @@ -952,7 +953,8 @@ mod _sqlite { parameters: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { - let cursor = Cursor::new(zelf.clone(), zelf.row_factory.to_owned(), vm).into_ref(vm); + let row_factory = zelf.row_factory.to_owned(); + let cursor = Cursor::new(zelf, row_factory, vm).into_ref(vm); Cursor::execute(cursor, sql, parameters, vm) } @@ -963,7 +965,8 @@ mod _sqlite { seq_of_params: ArgIterable, vm: &VirtualMachine, ) -> PyResult> { - let cursor = Cursor::new(zelf.clone(), zelf.row_factory.to_owned(), vm).into_ref(vm); + let row_factory = zelf.row_factory.to_owned(); + let cursor = Cursor::new(zelf, row_factory, vm).into_ref(vm); Cursor::executemany(cursor, sql, seq_of_params, vm) } @@ -973,15 +976,12 @@ mod _sqlite { script: PyStrRef, vm: &VirtualMachine, ) -> PyResult> { - Cursor::executescript( - Cursor::new(zelf.clone(), zelf.row_factory.to_owned(), vm).into_ref(vm), - script, - vm, - ) + let row_factory = zelf.row_factory.to_owned(); + Cursor::executescript(Cursor::new(zelf, row_factory, vm).into_ref(vm), script, vm) } #[pymethod] - fn backup(zelf: PyRef, args: BackupArgs, vm: &VirtualMachine) -> PyResult<()> { + fn backup(zelf: &Py, args: BackupArgs, vm: &VirtualMachine) -> PyResult<()> { let BackupArgs { target, pages, @@ -1550,8 +1550,8 @@ mod _sqlite { } #[pymethod] - fn fetchone(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - Self::next(&zelf, vm).map(|x| match x { + fn fetchone(zelf: &Py, vm: &VirtualMachine) -> PyResult { + Self::next(zelf, vm).map(|x| match x { PyIterReturn::Return(row) => row, PyIterReturn::StopIteration(_) => vm.ctx.none(), }) @@ -1559,13 +1559,13 @@ mod _sqlite { #[pymethod] fn fetchmany( - zelf: PyRef, + zelf: &Py, max_rows: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { let max_rows = max_rows.unwrap_or_else(|| zelf.arraysize.load(Ordering::Relaxed)); let mut list = vec![]; - while let PyIterReturn::Return(row) = Self::next(&zelf, vm)? { + while let PyIterReturn::Return(row) = Self::next(zelf, vm)? { list.push(row); if list.len() as c_int >= max_rows { break; @@ -1575,9 +1575,9 @@ mod _sqlite { } #[pymethod] - fn fetchall(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { + fn fetchall(zelf: &Py, vm: &VirtualMachine) -> PyResult> { let mut list = vec![]; - while let PyIterReturn::Return(row) = Self::next(&zelf, vm)? { + while let PyIterReturn::Return(row) = Self::next(zelf, vm)? { list.push(row); } Ok(list) From c8eff6ec079f45ebf86d4cc83f95a2e7861205db Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 00:55:58 +0900 Subject: [PATCH 11/20] clean up asyncgen --- vm/src/builtins/asyncgenerator.rs | 111 +++++++++++++++--------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/vm/src/builtins/asyncgenerator.rs b/vm/src/builtins/asyncgenerator.rs index b9b8493a1..5b4f6265a 100644 --- a/vm/src/builtins/asyncgenerator.rs +++ b/vm/src/builtins/asyncgenerator.rs @@ -26,7 +26,7 @@ impl PyPayload for PyAsyncGen { } } -#[pyclass(with(Constructor, Representable))] +#[pyclass(with(PyRef, Constructor, Representable))] impl PyAsyncGen { pub fn as_coro(&self) -> &Coro { &self.inner @@ -49,59 +49,6 @@ impl PyAsyncGen { self.inner.set_name(name) } - #[pymethod(magic)] - fn aiter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { - zelf - } - - #[pymethod(magic)] - fn anext(zelf: PyRef, vm: &VirtualMachine) -> PyAsyncGenASend { - Self::asend(zelf, vm.ctx.none(), vm) - } - - #[pymethod] - fn asend(zelf: PyRef, value: PyObjectRef, _vm: &VirtualMachine) -> PyAsyncGenASend { - PyAsyncGenASend { - ag: zelf, - state: AtomicCell::new(AwaitableState::Init), - value, - } - } - - #[pymethod] - fn athrow( - zelf: PyRef, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyAsyncGenAThrow { - PyAsyncGenAThrow { - ag: zelf, - aclose: false, - state: AtomicCell::new(AwaitableState::Init), - value: ( - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - ), - } - } - - #[pymethod] - fn aclose(zelf: PyRef, vm: &VirtualMachine) -> PyAsyncGenAThrow { - PyAsyncGenAThrow { - ag: zelf, - aclose: true, - state: AtomicCell::new(AwaitableState::Init), - value: ( - vm.ctx.exceptions.generator_exit.to_owned().into(), - vm.ctx.none(), - vm.ctx.none(), - ), - } - } - #[pygetset] fn ag_await(&self, _vm: &VirtualMachine) -> Option { self.inner.frame().yield_from_target() @@ -125,6 +72,62 @@ impl PyAsyncGen { } } +#[pyclass] +impl PyRef { + #[pymethod(magic)] + fn aiter(self, _vm: &VirtualMachine) -> PyRef { + self + } + + #[pymethod(magic)] + fn anext(self, vm: &VirtualMachine) -> PyAsyncGenASend { + Self::asend(self, vm.ctx.none(), vm) + } + + #[pymethod] + fn asend(self, value: PyObjectRef, _vm: &VirtualMachine) -> PyAsyncGenASend { + PyAsyncGenASend { + ag: self, + state: AtomicCell::new(AwaitableState::Init), + value, + } + } + + #[pymethod] + fn athrow( + self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyAsyncGenAThrow { + PyAsyncGenAThrow { + ag: self, + aclose: false, + state: AtomicCell::new(AwaitableState::Init), + value: ( + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + ), + } + } + + #[pymethod] + fn aclose(self, vm: &VirtualMachine) -> PyAsyncGenAThrow { + PyAsyncGenAThrow { + ag: self, + aclose: true, + state: AtomicCell::new(AwaitableState::Init), + value: ( + vm.ctx.exceptions.generator_exit.to_owned().into(), + vm.ctx.none(), + vm.ctx.none(), + ), + } + } +} + impl Representable for PyAsyncGen { #[inline] fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { From 914e14bb8deb87251fa43754654d4a61db86fc4e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 01:06:08 +0900 Subject: [PATCH 12/20] clean up bytearray --- vm/src/builtins/bytearray.rs | 435 +++++++++++++++++------------------ 1 file changed, 210 insertions(+), 225 deletions(-) diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index ad01049b5..a835d7a76 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -48,31 +48,6 @@ pub struct PyByteArray { pub type PyByteArrayRef = PyRef; -impl PyByteArray { - pub fn new_ref(data: Vec, ctx: &Context) -> PyRef { - PyRef::new_ref(Self::from(data), ctx.types.bytearray_type.to_owned(), None) - } - - fn from_inner(inner: PyBytesInner) -> Self { - PyByteArray { - inner: PyRwLock::new(inner), - exports: AtomicUsize::new(0), - } - } - - pub fn borrow_buf(&self) -> PyMappedRwLockReadGuard<'_, [u8]> { - PyRwLockReadGuard::map(self.inner.read(), |inner| &*inner.elements) - } - - pub fn borrow_buf_mut(&self) -> PyMappedRwLockWriteGuard<'_, Vec> { - PyRwLockWriteGuard::map(self.inner.write(), |inner| &mut inner.elements) - } - - fn repeat(&self, value: isize, vm: &VirtualMachine) -> PyResult { - self.inner().mul(value, vm).map(|x| x.into()) - } -} - impl From for PyByteArray { fn from(inner: PyBytesInner) -> Self { Self::from_inner(inner) @@ -97,9 +72,107 @@ pub(crate) fn init(context: &Context) { PyByteArrayIterator::extend_class(context, context.types.bytearray_iterator_type); } +impl PyByteArray { + pub fn new_ref(data: Vec, ctx: &Context) -> PyRef { + PyRef::new_ref(Self::from(data), ctx.types.bytearray_type.to_owned(), None) + } + + fn from_inner(inner: PyBytesInner) -> Self { + PyByteArray { + inner: PyRwLock::new(inner), + exports: AtomicUsize::new(0), + } + } + + pub fn borrow_buf(&self) -> PyMappedRwLockReadGuard<'_, [u8]> { + PyRwLockReadGuard::map(self.inner.read(), |inner| &*inner.elements) + } + + pub fn borrow_buf_mut(&self) -> PyMappedRwLockWriteGuard<'_, Vec> { + PyRwLockWriteGuard::map(self.inner.write(), |inner| &mut inner.elements) + } + + fn repeat(&self, value: isize, vm: &VirtualMachine) -> PyResult { + self.inner().mul(value, vm).map(|x| x.into()) + } + + fn _setitem_by_index(&self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let value = value_from_object(vm, &value)?; + self.borrow_buf_mut().setitem_by_index(vm, i, value) + } + + fn _setitem( + zelf: &Py, + needle: &PyObject, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + SequenceIndex::Int(i) => zelf._setitem_by_index(i, value, vm), + SequenceIndex::Slice(slice) => { + let items = if zelf.is(&value) { + zelf.borrow_buf().to_vec() + } else { + bytes_from_object(vm, &value)? + }; + if let Some(mut w) = zelf.try_resizable_opt() { + w.elements.setitem_by_slice(vm, slice, &items) + } else { + zelf.borrow_buf_mut() + .setitem_by_slice_no_resize(vm, slice, &items) + } + } + } + } + + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + SequenceIndex::Int(i) => self + .borrow_buf() + .getitem_by_index(vm, i) + .map(|x| vm.ctx.new_int(x).into()), + SequenceIndex::Slice(slice) => self + .borrow_buf() + .getitem_by_slice(vm, slice) + .map(|x| Self::new_ref(x, &vm.ctx).into()), + } + } + + pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i), + SequenceIndex::Slice(slice) => { + // TODO: delete 0 elements don't need resizable + self.try_resizable(vm)?.elements.delitem_by_slice(vm, slice) + } + } + } + + fn irepeat(zelf: &Py, n: isize, vm: &VirtualMachine) -> PyResult<()> { + if n == 1 { + return Ok(()); + } + let mut w = match zelf.try_resizable(vm) { + Ok(w) => w, + Err(err) => { + return if zelf.borrow_buf().is_empty() { + // We can multiple an empty vector by any integer + Ok(()) + } else { + Err(err) + }; + } + }; + + w.imul(n, vm) + } +} + #[pyclass( flags(BASETYPE), with( + Py, + PyRef, Constructor, Initializer, Comparable, @@ -156,45 +229,6 @@ impl PyByteArray { self.inner().contains(needle, vm) } - fn _setitem_by_index(&self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let value = value_from_object(vm, &value)?; - self.borrow_buf_mut().setitem_by_index(vm, i, value) - } - - fn _setitem( - zelf: PyRef, - needle: &PyObject, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { - SequenceIndex::Int(i) => zelf._setitem_by_index(i, value, vm), - SequenceIndex::Slice(slice) => { - let items = if zelf.is(&value) { - zelf.borrow_buf().to_vec() - } else { - bytes_from_object(vm, &value)? - }; - if let Some(mut w) = zelf.try_resizable_opt() { - w.elements.setitem_by_slice(vm, slice, &items) - } else { - zelf.borrow_buf_mut() - .setitem_by_slice_no_resize(vm, slice, &items) - } - } - } - } - - #[pymethod(magic)] - fn setitem( - zelf: PyRef, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::_setitem(zelf, &needle, value, vm) - } - #[pymethod(magic)] fn iadd(zelf: PyRef, other: ArgBytesLike, vm: &VirtualMachine) -> PyResult> { zelf.try_resizable(vm)? @@ -203,34 +237,11 @@ impl PyByteArray { Ok(zelf) } - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { - SequenceIndex::Int(i) => self - .borrow_buf() - .getitem_by_index(vm, i) - .map(|x| vm.ctx.new_int(x).into()), - SequenceIndex::Slice(slice) => self - .borrow_buf() - .getitem_by_slice(vm, slice) - .map(|x| Self::new_ref(x, &vm.ctx).into()), - } - } - #[pymethod(magic)] fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } - pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { - SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i), - SequenceIndex::Slice(slice) => { - // TODO: delete 0 elements don't need resizable - self.try_resizable(vm)?.elements.delitem_by_slice(vm, slice) - } - } - } - #[pymethod(magic)] pub fn delitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self._delitem(&needle, vm) @@ -241,78 +252,6 @@ impl PyByteArray { PyBytesInner::maketrans(from, to, vm) } - #[pymethod] - fn pop(zelf: PyRef, index: OptionalArg, vm: &VirtualMachine) -> PyResult { - let elements = &mut zelf.try_resizable(vm)?.elements; - let index = elements - .wrap_index(index.unwrap_or(-1)) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; - Ok(elements.remove(index)) - } - - #[pymethod] - fn insert( - zelf: PyRef, - index: isize, - object: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let value = value_from_object(vm, &object)?; - let elements = &mut zelf.try_resizable(vm)?.elements; - let index = elements.saturate_index(index); - elements.insert(index, value); - Ok(()) - } - - #[pymethod] - fn append(zelf: PyRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let value = value_from_object(vm, &object)?; - zelf.try_resizable(vm)?.elements.push(value); - Ok(()) - } - - #[pymethod] - fn remove(zelf: PyRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let value = value_from_object(vm, &object)?; - let elements = &mut zelf.try_resizable(vm)?.elements; - if let Some(index) = elements.find_byte(value) { - elements.remove(index); - Ok(()) - } else { - Err(vm.new_value_error("value not found in bytearray".to_owned())) - } - } - - #[pymethod] - fn extend(zelf: PyRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if zelf.is(&object) { - Self::irepeat(&zelf, 2, vm) - } else { - let items = bytes_from_object(vm, &object)?; - zelf.try_resizable(vm)?.elements.extend(items); - Ok(()) - } - } - - fn irepeat(zelf: &Py, n: isize, vm: &VirtualMachine) -> PyResult<()> { - if n == 1 { - return Ok(()); - } - let mut w = match zelf.try_resizable(vm) { - Ok(w) => w, - Err(err) => { - return if zelf.borrow_buf().is_empty() { - // We can multiple an empty vector by any integer - Ok(()) - } else { - Err(err) - }; - } - }; - - w.imul(n, vm) - } - #[pymethod] fn isalnum(&self) -> bool { self.inner().isalnum() @@ -503,59 +442,11 @@ impl PyByteArray { self.inner().strip(chars).into() } - #[pymethod] - fn lstrip( - zelf: PyRef, - chars: OptionalOption, - vm: &VirtualMachine, - ) -> PyRef { - let inner = zelf.inner(); - let stripped = inner.lstrip(chars); - let elements = &inner.elements; - if stripped == elements { - drop(inner); - zelf - } else { - vm.new_pyref(PyByteArray::from(stripped.to_vec())) - } - } - - #[pymethod] - fn rstrip( - zelf: PyRef, - chars: OptionalOption, - vm: &VirtualMachine, - ) -> PyRef { - let inner = zelf.inner(); - let stripped = inner.rstrip(chars); - let elements = &inner.elements; - if stripped == elements { - drop(inner); - zelf - } else { - vm.new_pyref(PyByteArray::from(stripped.to_vec())) - } - } - - /// removeprefix($self, prefix, /) - /// - /// - /// Return a bytearray object with the given prefix string removed if present. - /// - /// If the bytearray starts with the prefix string, return string[len(prefix):] - /// Otherwise, return a copy of the original bytearray. #[pymethod] fn removeprefix(&self, prefix: PyBytesInner) -> Self { self.inner().removeprefix(prefix).into() } - /// removesuffix(self, prefix, /) - /// - /// - /// Return a bytearray object with the given suffix string removed if present. - /// - /// If the bytearray ends with the suffix string, return string[:len(suffix)] - /// Otherwise, return a copy of the original bytearray. #[pymethod] fn removesuffix(&self, suffix: PyBytesInner) -> Self { self.inner().removesuffix(suffix).to_vec().into() @@ -638,12 +529,6 @@ impl PyByteArray { Ok(self.inner().replace(old, new, count, vm)?.into()) } - #[pymethod] - fn clear(zelf: PyRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.try_resizable(vm)?.elements.clear(); - Ok(()) - } - #[pymethod] fn copy(&self) -> Self { self.borrow_buf().to_vec().into() @@ -681,35 +566,135 @@ impl PyByteArray { fn reverse(&self) { self.borrow_buf_mut().reverse(); } +} + +#[pyclass] +impl Py { + #[pymethod(magic)] + fn setitem( + &self, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + PyByteArray::_setitem(self, &needle, value, vm) + } #[pymethod] - fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { - bytes_decode(zelf.into(), args, vm) + fn pop(&self, index: OptionalArg, vm: &VirtualMachine) -> PyResult { + let elements = &mut self.try_resizable(vm)?.elements; + let index = elements + .wrap_index(index.unwrap_or(-1)) + .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + Ok(elements.remove(index)) + } + + #[pymethod] + fn insert(&self, index: isize, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let value = value_from_object(vm, &object)?; + let elements = &mut self.try_resizable(vm)?.elements; + let index = elements.saturate_index(index); + elements.insert(index, value); + Ok(()) + } + + #[pymethod] + fn append(&self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let value = value_from_object(vm, &object)?; + self.try_resizable(vm)?.elements.push(value); + Ok(()) + } + + #[pymethod] + fn remove(&self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let value = value_from_object(vm, &object)?; + let elements = &mut self.try_resizable(vm)?.elements; + let index = elements + .find_byte(value) + .ok_or_else(|| vm.new_value_error("value not found in bytearray".to_owned()))?; + elements.remove(index); + Ok(()) + } + + #[pymethod] + fn extend(&self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if self.is(&object) { + PyByteArray::irepeat(self, 2, vm) + } else { + let items = bytes_from_object(vm, &object)?; + self.try_resizable(vm)?.elements.extend(items); + Ok(()) + } + } + + #[pymethod] + fn clear(&self, vm: &VirtualMachine) -> PyResult<()> { + self.try_resizable(vm)?.elements.clear(); + Ok(()) } #[pymethod(magic)] fn reduce_ex( - zelf: PyRef, + &self, _proto: usize, vm: &VirtualMachine, ) -> (PyTypeRef, PyTupleRef, Option) { - Self::reduce(zelf, vm) + Self::reduce(self, vm) } #[pymethod(magic)] - fn reduce( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, PyTupleRef, Option) { - let bytes = PyBytes::from(zelf.borrow_buf().to_vec()).to_pyobject(vm); + fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option) { + let bytes = PyBytes::from(self.borrow_buf().to_vec()).to_pyobject(vm); ( - zelf.class().to_owned(), + self.class().to_owned(), PyTuple::new_ref(vec![bytes], &vm.ctx), - zelf.as_object().dict(), + self.as_object().dict(), ) } } +#[pyclass] +impl PyRef { + #[pymethod] + fn lstrip( + self, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let inner = self.inner(); + let stripped = inner.lstrip(chars); + let elements = &inner.elements; + if stripped == elements { + drop(inner); + self + } else { + vm.new_pyref(PyByteArray::from(stripped.to_vec())) + } + } + + #[pymethod] + fn rstrip( + self, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let inner = self.inner(); + let stripped = inner.rstrip(chars); + let elements = &inner.elements; + if stripped == elements { + drop(inner); + self + } else { + vm.new_pyref(PyByteArray::from(stripped.to_vec())) + } + } + + #[pymethod] + fn decode(self, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(self.into(), args, vm) + } +} + impl Constructor for PyByteArray { type Args = FuncArgs; @@ -796,7 +781,7 @@ impl AsMapping for PyByteArray { ass_subscript: atomic_func!(|mapping, needle, value, vm| { let zelf = PyByteArray::mapping_downcast(mapping); if let Some(value) = value { - PyByteArray::setitem(zelf.to_owned(), needle.to_owned(), value, vm) + Py::setitem(zelf, needle.to_owned(), value, vm) } else { zelf.delitem(needle.to_owned(), vm) } From 1db415d8974c6327acf082f41722e69daf20cd76 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 01:46:15 +0900 Subject: [PATCH 13/20] clean up bytes --- vm/src/builtins/bytes.rs | 190 ++++++++++++++++++--------------------- 1 file changed, 87 insertions(+), 103 deletions(-) diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index f32f78422..5715d3f3b 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -101,23 +101,38 @@ impl PyBytes { PyRef::new_ref(Self::from(data), ctx.types.bytes_type.to_owned(), None) } - fn repeat(zelf: PyRef, count: isize, vm: &VirtualMachine) -> PyResult> { - if count == 1 && zelf.class().is(vm.ctx.types.bytes_type) { + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object(vm, needle, "byte")? { + SequenceIndex::Int(i) => self + .getitem_by_index(vm, i) + .map(|x| vm.ctx.new_int(x).into()), + SequenceIndex::Slice(slice) => self + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_bytes(x).into()), + } + } +} + +impl PyRef { + fn repeat(self, count: isize, vm: &VirtualMachine) -> PyResult> { + if count == 1 && self.class().is(vm.ctx.types.bytes_type) { // Special case: when some `bytes` is multiplied by `1`, // nothing really happens, we need to return an object itself // with the same `id()` to be compatible with CPython. // This only works for `bytes` itself, not its subclasses. - return Ok(zelf); + return Ok(self); } - zelf.inner + self.inner .mul(count, vm) - .map(|x| Self::from(x).into_ref(vm)) + .map(|x| PyBytes::from(x).into_ref(vm)) } } #[pyclass( flags(BASETYPE), with( + Py, + PyRef, AsMapping, AsSequence, Hashable, @@ -146,15 +161,6 @@ impl PyBytes { self.inner.as_bytes() } - #[pymethod(magic)] - fn bytes(zelf: PyRef, vm: &VirtualMachine) -> PyRef { - if zelf.is(vm.ctx.types.bytes_type) { - zelf - } else { - PyBytes::from(zelf.inner.clone()).into_ref(vm) - } - } - #[pymethod(magic)] fn sizeof(&self) -> usize { size_of::() + self.len() * size_of::() @@ -179,17 +185,6 @@ impl PyBytes { PyBytesInner::maketrans(from, to, vm) } - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "byte")? { - SequenceIndex::Int(i) => self - .getitem_by_index(vm, i) - .map(|x| vm.ctx.new_int(x).into()), - SequenceIndex::Slice(slice) => self - .getitem_by_slice(vm, slice) - .map(|x| vm.ctx.new_bytes(x).into()), - } - } - #[pymethod(magic)] fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) @@ -371,53 +366,11 @@ impl PyBytes { self.inner.strip(chars).into() } - #[pymethod] - fn lstrip( - zelf: PyRef, - chars: OptionalOption, - vm: &VirtualMachine, - ) -> PyRef { - let stripped = zelf.inner.lstrip(chars); - if stripped == zelf.as_bytes() { - zelf - } else { - vm.ctx.new_bytes(stripped.to_vec()) - } - } - - #[pymethod] - fn rstrip( - zelf: PyRef, - chars: OptionalOption, - vm: &VirtualMachine, - ) -> PyRef { - let stripped = zelf.inner.rstrip(chars); - if stripped == zelf.as_bytes() { - zelf - } else { - vm.ctx.new_bytes(stripped.to_vec()) - } - } - - /// removeprefix($self, prefix, /) - /// - /// - /// Return a bytes object with the given prefix string removed if present. - /// - /// If the bytes starts with the prefix string, return string[len(prefix):] - /// Otherwise, return a copy of the original bytes. #[pymethod] fn removeprefix(&self, prefix: PyBytesInner) -> Self { self.inner.removeprefix(prefix).into() } - /// removesuffix(self, prefix, /) - /// - /// - /// Return a bytes object with the given suffix string removed if present. - /// - /// If the bytes ends with the suffix string, return string[:len(suffix)] - /// Otherwise, return a copy of the original bytes. #[pymethod] fn removesuffix(&self, suffix: PyBytesInner) -> Self { self.inner.removesuffix(suffix).into() @@ -508,7 +461,7 @@ impl PyBytes { #[pymethod(name = "__rmul__")] #[pymethod(magic)] fn mul(zelf: PyRef, value: ArgIndex, vm: &VirtualMachine) -> PyResult> { - Self::repeat(zelf, value.try_to_primitive(vm)?, vm) + zelf.repeat(value.try_to_primitive(vm)?, vm) } #[pymethod(name = "__mod__")] @@ -522,6 +475,66 @@ impl PyBytes { vm.ctx.not_implemented() } + #[pymethod(magic)] + fn getnewargs(&self, vm: &VirtualMachine) -> PyTupleRef { + let param: Vec = self.elements().map(|x| x.to_pyobject(vm)).collect(); + PyTuple::new_ref(param, &vm.ctx) + } +} + +#[pyclass] +impl Py { + #[pymethod(magic)] + fn reduce_ex( + &self, + _proto: usize, + vm: &VirtualMachine, + ) -> (PyTypeRef, PyTupleRef, Option) { + Self::reduce(self, vm) + } + + #[pymethod(magic)] + fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option) { + let bytes = PyBytes::from(self.to_vec()).to_pyobject(vm); + ( + self.class().to_owned(), + PyTuple::new_ref(vec![bytes], &vm.ctx), + self.as_object().dict(), + ) + } +} + +#[pyclass] +impl PyRef { + #[pymethod(magic)] + fn bytes(self, vm: &VirtualMachine) -> PyRef { + if self.is(vm.ctx.types.bytes_type) { + self + } else { + PyBytes::from(self.inner.clone()).into_ref(vm) + } + } + + #[pymethod] + fn lstrip(self, chars: OptionalOption, vm: &VirtualMachine) -> PyRef { + let stripped = self.inner.lstrip(chars); + if stripped == self.as_bytes() { + self + } else { + vm.ctx.new_bytes(stripped.to_vec()) + } + } + + #[pymethod] + fn rstrip(self, chars: OptionalOption, vm: &VirtualMachine) -> PyRef { + let stripped = self.inner.rstrip(chars); + if stripped == self.as_bytes() { + self + } else { + vm.ctx.new_bytes(stripped.to_vec()) + } + } + /// Return a string decoded from the given bytes. /// Default encoding is 'utf-8'. /// Default errors is 'strict', meaning that encoding errors raise a UnicodeError. @@ -530,36 +543,8 @@ impl PyBytes { /// see https://docs.python.org/3/library/codecs.html#standard-encodings /// currently, only 'utf-8' and 'ascii' emplemented #[pymethod] - fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { - bytes_decode(zelf.into(), args, vm) - } - - #[pymethod(magic)] - fn getnewargs(&self, vm: &VirtualMachine) -> PyTupleRef { - let param: Vec = self.elements().map(|x| x.to_pyobject(vm)).collect(); - PyTuple::new_ref(param, &vm.ctx) - } - - #[pymethod(magic)] - fn reduce_ex( - zelf: PyRef, - _proto: usize, - vm: &VirtualMachine, - ) -> (PyTypeRef, PyTupleRef, Option) { - Self::reduce(zelf, vm) - } - - #[pymethod(magic)] - fn reduce( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, PyTupleRef, Option) { - let bytes = PyBytes::from(zelf.to_vec()).to_pyobject(vm); - ( - zelf.class().to_owned(), - PyTuple::new_ref(vec![bytes], &vm.ctx), - zelf.as_object().dict(), - ) + fn decode(self, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(self.into(), args, vm) } } @@ -605,11 +590,10 @@ impl AsSequence for PyBytes { .map(|x| vm.ctx.new_bytes(x).into()) }), repeat: atomic_func!(|seq, n, vm| { - if let Ok(zelf) = seq.obj.to_owned().downcast::() { - PyBytes::repeat(zelf, n, vm).to_pyresult(vm) - } else { - Err(vm.new_type_error("bad argument type for built-in operation".to_owned())) - } + let zelf = seq.obj.to_owned().downcast::().map_err(|_| { + vm.new_type_error("bad argument type for built-in operation".to_owned()) + })?; + zelf.repeat(n, vm).to_pyresult(vm) }), item: atomic_func!(|seq, i, vm| { PyBytes::sequence_downcast(seq) From 8fd984a7672bd0876079b86ace35cb2b3bdcfc32 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 01:51:45 +0900 Subject: [PATCH 14/20] clean up coroutine --- vm/src/builtins/coroutine.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/builtins/coroutine.rs b/vm/src/builtins/coroutine.rs index 7c400274e..76d15b3ae 100644 --- a/vm/src/builtins/coroutine.rs +++ b/vm/src/builtins/coroutine.rs @@ -45,13 +45,13 @@ impl PyCoroutine { } #[pymethod] - fn send(zelf: PyRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn send(zelf: &Py, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { zelf.inner.send(zelf.as_object(), value, vm) } #[pymethod] fn throw( - zelf: PyRef, + zelf: &Py, exc_type: PyObjectRef, exc_val: OptionalArg, exc_tb: OptionalArg, @@ -67,7 +67,7 @@ impl PyCoroutine { } #[pymethod] - fn close(zelf: PyRef, vm: &VirtualMachine) -> PyResult<()> { + fn close(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { zelf.inner.close(zelf.as_object(), vm) } @@ -111,7 +111,7 @@ impl Representable for PyCoroutine { impl IterNextIterable for PyCoroutine {} impl IterNext for PyCoroutine { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - Self::send(zelf.to_owned(), vm.ctx.none(), vm) + Self::send(zelf, vm.ctx.none(), vm) } } @@ -131,8 +131,8 @@ impl PyPayload for PyCoroutineWrapper { #[pyclass(with(IterNext))] impl PyCoroutineWrapper { #[pymethod] - fn send(zelf: PyRef, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - PyCoroutine::send(zelf.coro.clone(), val, vm) + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + PyCoroutine::send(&self.coro, val, vm) } #[pymethod] @@ -143,14 +143,14 @@ impl PyCoroutineWrapper { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PyCoroutine::throw(self.coro.clone(), exc_type, exc_val, exc_tb, vm) + PyCoroutine::throw(&self.coro, exc_type, exc_val, exc_tb, vm) } } impl IterNextIterable for PyCoroutineWrapper {} impl IterNext for PyCoroutineWrapper { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - Self::send(zelf.to_owned(), vm.ctx.none(), vm) + Self::send(zelf, vm.ctx.none(), vm) } } From cf82d6f0a54076b0cc06b2812a41329ada9b12d6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 01:55:20 +0900 Subject: [PATCH 15/20] clean up descriptor --- vm/src/builtins/descriptor.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/builtins/descriptor.rs b/vm/src/builtins/descriptor.rs index 8d4154f0c..8149c3065 100644 --- a/vm/src/builtins/descriptor.rs +++ b/vm/src/builtins/descriptor.rs @@ -3,7 +3,7 @@ use crate::{ class::PyClassImpl, function::PySetterValue, types::{Constructor, GetDescriptor, Representable, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use rustpython_common::lock::PyRwLock; @@ -103,8 +103,8 @@ fn calculate_qualname(descr: &DescrObject, vm: &VirtualMachine) -> PyResult) -> Option { - zelf.member.doc.to_owned() + fn doc(&self) -> Option { + self.member.doc.to_owned() } #[pygetset(magic)] From 29073cf711a2c3a07c6ebf9133796a131a63950f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 02:09:41 +0900 Subject: [PATCH 16/20] clean up dict --- vm/src/builtins/dict.rs | 70 +++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index 6b84c403f..74d71ddac 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -203,6 +203,8 @@ impl PyDict { #[allow(clippy::len_without_is_empty)] #[pyclass( with( + Py, + PyRef, Constructor, Initializer, AsMapping, @@ -270,32 +272,11 @@ impl PyDict { self.entries.clear() } - #[pymethod] - fn keys(zelf: PyRef) -> PyDictKeys { - PyDictKeys::new(zelf) - } - - #[pymethod] - fn values(zelf: PyRef) -> PyDictValues { - PyDictValues::new(zelf) - } - - #[pymethod] - fn items(zelf: PyRef) -> PyDictItems { - PyDictItems::new(zelf) - } - #[pymethod(magic)] fn setitem(&self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner_setitem(&*key, value, vm) } - #[pymethod(magic)] - #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] - fn getitem(zelf: PyRef, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - zelf.inner_getitem(&*key, vm) - } - #[pymethod] fn get( &self, @@ -396,17 +377,44 @@ impl PyDict { Ok((key, value)) } - #[pymethod(magic)] - fn reversed(zelf: PyRef) -> PyDictReverseKeyIterator { - PyDictReverseKeyIterator::new(zelf) - } - #[pyclassmethod(magic)] fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { PyGenericAlias::new(cls, args, vm) } } +#[pyclass] +impl Py { + #[pymethod(magic)] + #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] + fn getitem(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner_getitem(&*key, vm) + } +} + +#[pyclass] +impl PyRef { + #[pymethod] + fn keys(self) -> PyDictKeys { + PyDictKeys::new(self) + } + + #[pymethod] + fn values(self) -> PyDictValues { + PyDictValues::new(self) + } + + #[pymethod] + fn items(self) -> PyDictItems { + PyDictItems::new(self) + } + + #[pymethod(magic)] + fn reversed(self) -> PyDictReverseKeyIterator { + PyDictReverseKeyIterator::new(self) + } +} + impl Constructor for PyDict { type Args = FuncArgs; @@ -847,9 +855,9 @@ macro_rules! dict_view { #[allow(clippy::redundant_closure_call)] #[pymethod(magic)] - fn reduce(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { + fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { let iter = builtins_iter(vm).to_owned(); - let internal = zelf.internal.lock(); + let internal = self.internal.lock(); let entries = match &internal.status { IterStatus::Active(dict) => dict .into_iter() @@ -917,9 +925,9 @@ macro_rules! dict_view { #[allow(clippy::redundant_closure_call)] #[pymethod(magic)] - fn reduce(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { + fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { let iter = builtins_reversed(vm).to_owned(); - let internal = zelf.internal.lock(); + let internal = self.internal.lock(); // TODO: entries must be reversed too let entries = match &internal.status { IterStatus::Active(dict) => dict @@ -1240,7 +1248,7 @@ impl PyDictItems { return Ok(false); } let value = needle.fast_getitem(1); - let found = PyDict::getitem(zelf.dict().clone(), key, vm)?; + let found = zelf.dict().getitem(key, vm)?; vm.identical_or_equal(&found, &value) } #[pygetset] From 29055ab4a60814db0ac975fd5005d8fa2864e306 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 02:12:10 +0900 Subject: [PATCH 17/20] clean up enumerate --- vm/src/builtins/enumerate.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vm/src/builtins/enumerate.rs b/vm/src/builtins/enumerate.rs index f07f7d612..2905d4353 100644 --- a/vm/src/builtins/enumerate.rs +++ b/vm/src/builtins/enumerate.rs @@ -8,7 +8,7 @@ use crate::{ function::OptionalArg, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, IterNextIterable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use num_bigint::BigInt; use num_traits::Zero; @@ -51,17 +51,21 @@ impl Constructor for PyEnumerate { } } -#[pyclass(with(IterNext, Constructor), flags(BASETYPE))] +#[pyclass(with(Py, IterNext, Constructor), flags(BASETYPE))] impl PyEnumerate { #[pyclassmethod(magic)] fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { PyGenericAlias::new(cls, args, vm) } +} + +#[pyclass] +impl Py { #[pymethod(magic)] - fn reduce(zelf: PyRef) -> (PyTypeRef, (PyIter, BigInt)) { + fn reduce(&self) -> (PyTypeRef, (PyIter, BigInt)) { ( - zelf.class().to_owned(), - (zelf.iterator.clone(), zelf.counter.read().clone()), + self.class().to_owned(), + (self.iterator.clone(), self.counter.read().clone()), ) } } From 717242636a5208fa7cbf99d6cd2f85d0ce95035c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 02:18:52 +0900 Subject: [PATCH 18/20] clean up generator --- vm/src/builtins/generator.rs | 61 +++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/vm/src/builtins/generator.rs b/vm/src/builtins/generator.rs index 24ae1994a..36fd488af 100644 --- a/vm/src/builtins/generator.rs +++ b/vm/src/builtins/generator.rs @@ -25,7 +25,7 @@ impl PyPayload for PyGenerator { } } -#[pyclass(with(Constructor, IterNext))] +#[pyclass(with(Py, Constructor, IterNext))] impl PyGenerator { pub fn as_coro(&self) -> &Coro { &self.inner @@ -47,33 +47,6 @@ impl PyGenerator { self.inner.set_name(name) } - #[pymethod] - fn send(zelf: PyRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - zelf.inner.send(zelf.as_object(), value, vm) - } - - #[pymethod] - fn throw( - zelf: PyRef, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - zelf.inner.throw( - zelf.as_object(), - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - vm, - ) - } - - #[pymethod] - fn close(zelf: PyRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.inner.close(zelf.as_object(), vm) - } - #[pygetset] fn gi_frame(&self, _vm: &VirtualMachine) -> FrameRef { self.inner.frame() @@ -92,6 +65,36 @@ impl PyGenerator { } } +#[pyclass] +impl Py { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.send(self.as_object(), value, vm) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.throw( + self.as_object(), + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + vm, + ) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + self.inner.close(self.as_object(), vm) + } +} + impl Unconstructible for PyGenerator {} impl Representable for PyGenerator { @@ -104,7 +107,7 @@ impl Representable for PyGenerator { impl IterNextIterable for PyGenerator {} impl IterNext for PyGenerator { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - Self::send(zelf.to_owned(), vm.ctx.none(), vm) + zelf.send(vm.ctx.none(), vm) } } From 962b3659c6700af07961bc4c0e681a9a70b6b35d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 02:20:13 +0900 Subject: [PATCH 19/20] clean up genericalias --- vm/src/builtins/genericalias.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 8d8b4d92f..fa745cdb1 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -179,7 +179,7 @@ impl PyGenericAlias { } #[pymethod(magic)] - fn reduce(zelf: PyRef, vm: &VirtualMachine) -> (PyTypeRef, (PyTypeRef, PyTupleRef)) { + fn reduce(zelf: &Py, vm: &VirtualMachine) -> (PyTypeRef, (PyTypeRef, PyTupleRef)) { ( vm.ctx.types.generic_alias_type.to_owned(), (zelf.origin.clone(), zelf.args.clone()), From cf2b48fc0c1e7caadbb696f8ee41bc7b6bc8f203 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Mar 2023 02:30:38 +0900 Subject: [PATCH 20/20] clean up memory --- vm/src/builtins/memory.rs | 831 +++++++++++++++++++------------------- 1 file changed, 419 insertions(+), 412 deletions(-) diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 81576b598..0b944eb4c 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -68,16 +68,6 @@ impl Constructor for PyMemoryView { } } -#[pyclass(with( - Hashable, - Comparable, - AsBuffer, - AsMapping, - AsSequence, - Constructor, - Iterable, - Representable -))] impl PyMemoryView { fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult { FormatSpec::parse(format.as_bytes(), vm) @@ -143,13 +133,6 @@ impl PyMemoryView { zelf } - #[pymethod] - pub fn release(&self) { - if self.released.compare_exchange(false, true).is_ok() { - self.buffer.release(); - } - } - fn try_not_released(&self, vm: &VirtualMachine) -> PyResult<()> { if self.released.load() { Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) @@ -158,100 +141,6 @@ impl PyMemoryView { } } - #[pygetset] - fn obj(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.buffer.obj.clone()) - } - - #[pygetset] - fn nbytes(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.len) - } - - #[pygetset] - fn readonly(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.readonly) - } - - #[pygetset] - fn itemsize(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.itemsize) - } - - #[pygetset] - fn ndim(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.ndim()) - } - - #[pygetset] - fn shape(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm)?; - Ok(vm.ctx.new_tuple( - self.desc - .dim_desc - .iter() - .map(|(shape, _, _)| shape.to_pyobject(vm)) - .collect(), - )) - } - - #[pygetset] - fn strides(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm)?; - Ok(vm.ctx.new_tuple( - self.desc - .dim_desc - .iter() - .map(|(_, stride, _)| stride.to_pyobject(vm)) - .collect(), - )) - } - - #[pygetset] - fn suboffsets(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm)?; - Ok(vm.ctx.new_tuple( - self.desc - .dim_desc - .iter() - .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) - .collect(), - )) - } - - #[pygetset] - fn format(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm) - .map(|_| PyStr::from(self.desc.format.clone())) - } - - #[pygetset] - fn contiguous(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.is_contiguous()) - } - - #[pygetset] - fn c_contiguous(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.is_contiguous()) - } - - #[pygetset] - fn f_contiguous(&self, vm: &VirtualMachine) -> PyResult { - // TODO: fortain order - self.try_not_released(vm) - .map(|_| self.desc.ndim() <= 1 && self.desc.is_contiguous()) - } - - #[pymethod(magic)] - fn enter(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { - zelf.try_not_released(vm).map(|_| zelf) - } - - #[pymethod(magic)] - fn exit(&self, _args: FuncArgs) { - self.release(); - } - fn getitem_by_idx(&self, i: isize, vm: &VirtualMachine) -> PyResult { if self.desc.ndim() != 1 { return Err(vm.new_not_implemented_error( @@ -281,29 +170,6 @@ impl PyMemoryView { format_unpack(&self.format_spec, &bytes[pos..pos + self.desc.itemsize], vm) } - #[pymethod(magic)] - fn getitem(zelf: PyRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - zelf.try_not_released(vm)?; - if zelf.desc.ndim() == 0 { - // 0-d memoryview can be referenced using mv[...] or mv[()] only - if needle.is(&vm.ctx.ellipsis) { - return Ok(zelf.into()); - } - if let Some(tuple) = needle.payload::() { - if tuple.is_empty() { - return zelf.unpack_single(0, vm); - } - } - return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); - } - - match SubscriptNeedle::try_from_object(vm, needle)? { - SubscriptNeedle::Index(i) => zelf.getitem_by_idx(i, vm), - SubscriptNeedle::Slice(slice) => zelf.getitem_by_slice(&slice, vm), - SubscriptNeedle::MultiIndex(indices) => zelf.getitem_by_multi_idx(&indices, vm), - } - } - fn setitem_by_idx(&self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.desc.ndim() != 1 { return Err(vm.new_not_implemented_error("sub-views are not implemented".to_owned())); @@ -317,60 +183,6 @@ impl PyMemoryView { self.pack_single(pos, value, vm) } - fn setitem_by_slice( - zelf: PyRef, - slice: &PySlice, - src: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if zelf.desc.ndim() != 1 { - return Err(vm.new_not_implemented_error("sub-view are not implemented".to_owned())); - } - - let mut dest = zelf.new_view(); - dest.init_slice(slice, 0, vm)?; - dest.init_len(); - - if zelf.is(&src) { - return if !is_equiv_structure(&zelf.desc, &dest.desc) { - Err(vm.new_value_error( - "memoryview assignment: lvalue and rvalue have different structures".to_owned(), - )) - } else { - // assign self[:] to self - Ok(()) - }; - }; - - let src = if let Some(src) = src.downcast_ref::() { - if zelf.buffer.obj.is(&src.buffer.obj) { - src.to_contiguous(vm) - } else { - AsBuffer::as_buffer(src, vm)? - } - } else { - PyBuffer::try_from_object(vm, src)? - }; - - if !is_equiv_structure(&src.desc, &dest.desc) { - return Err(vm.new_value_error( - "memoryview assignment: lvalue and rvalue have different structures".to_owned(), - )); - } - - let mut bytes_mut = dest.buffer.obj_bytes_mut(); - let src_bytes = src.obj_bytes(); - dest.desc.zip_eq(&src.desc, true, |a_range, b_range| { - let a_range = (a_range.start + dest.start as isize) as usize - ..(a_range.end + dest.start as isize) as usize; - let b_range = b_range.start as usize..b_range.end as usize; - bytes_mut[a_range].copy_from_slice(&src_bytes[b_range]); - false - }); - - Ok(()) - } - fn setitem_by_multi_idx( &self, indexes: &[isize], @@ -381,47 +193,6 @@ impl PyMemoryView { self.pack_single(pos, value, vm) } - #[pymethod(magic)] - fn delitem(&self, _needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if self.desc.readonly { - return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); - } - Err(vm.new_type_error("cannot delete memory".to_owned())) - } - - #[pymethod(magic)] - fn setitem( - zelf: PyRef, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.try_not_released(vm)?; - if zelf.desc.readonly { - return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); - } - if value.is(&vm.ctx.none) { - return Err(vm.new_type_error("cannot delete memory".to_owned())); - } - - if zelf.desc.ndim() == 0 { - // TODO: merge branches when we got conditional if let - if needle.is(&vm.ctx.ellipsis) { - return zelf.pack_single(0, value, vm); - } else if let Some(tuple) = needle.payload::() { - if tuple.is_empty() { - return zelf.pack_single(0, value, vm); - } - } - return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); - } - match SubscriptNeedle::try_from_object(vm, needle)? { - SubscriptNeedle::Index(i) => zelf.setitem_by_idx(i, value, vm), - SubscriptNeedle::Slice(slice) => Self::setitem_by_slice(zelf, &slice, value, vm), - SubscriptNeedle::MultiIndex(indices) => zelf.setitem_by_multi_idx(&indices, value, vm), - } - } - fn pack_single(&self, pos: usize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let mut bytes = self.buffer.obj_bytes_mut(); // TODO: Optimize @@ -522,26 +293,6 @@ impl PyMemoryView { Ok(()) } - /// return the length of the first dimension - #[pymethod(magic)] - fn len(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm)?; - Ok(if self.desc.ndim() == 0 { - 1 - } else { - // shape for dim[0] - self.desc.dim_desc[0].0 - }) - } - - #[pymethod] - fn tobytes(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm)?; - let mut v = vec![]; - self.append_to(&mut v); - Ok(PyBytes::from(v).into_ref(vm)) - } - fn _to_list( &self, bytes: &[u8], @@ -572,6 +323,389 @@ impl PyMemoryView { Ok(vm.ctx.new_list(v)) } + fn eq(zelf: &Py, other: &PyObject, vm: &VirtualMachine) -> PyResult { + if zelf.is(other) { + return Ok(true); + } + if zelf.released.load() { + return Ok(false); + } + + if let Some(other) = other.payload::() { + if other.released.load() { + return Ok(false); + } + } + + let other = match PyBuffer::try_from_borrowed_object(vm, other) { + Ok(buf) => buf, + Err(_) => return Ok(false), + }; + + if !is_equiv_shape(&zelf.desc, &other.desc) { + return Ok(false); + } + + let a_itemsize = zelf.desc.itemsize; + let b_itemsize = other.desc.itemsize; + let a_format_spec = &zelf.format_spec; + let b_format_spec = &Self::parse_format(&other.desc.format, vm)?; + + if zelf.desc.ndim() == 0 { + let a_val = format_unpack(a_format_spec, &zelf.buffer.obj_bytes()[..a_itemsize], vm)?; + let b_val = format_unpack(b_format_spec, &other.obj_bytes()[..b_itemsize], vm)?; + return vm.bool_eq(&a_val, &b_val); + } + + // TODO: optimize cmp by format + let mut ret = Ok(true); + let a_bytes = zelf.buffer.obj_bytes(); + let b_bytes = other.obj_bytes(); + zelf.desc.zip_eq(&other.desc, false, |a_range, b_range| { + let a_range = (a_range.start + zelf.start as isize) as usize + ..(a_range.end + zelf.start as isize) as usize; + let b_range = b_range.start as usize..b_range.end as usize; + let a_val = match format_unpack(a_format_spec, &a_bytes[a_range], vm) { + Ok(val) => val, + Err(e) => { + ret = Err(e); + return true; + } + }; + let b_val = match format_unpack(b_format_spec, &b_bytes[b_range], vm) { + Ok(val) => val, + Err(e) => { + ret = Err(e); + return true; + } + }; + ret = vm.bool_eq(&a_val, &b_val); + if let Ok(b) = ret { + !b + } else { + true + } + }); + ret + } + + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + if self.desc.is_contiguous() { + BorrowedValue::map(self.buffer.obj_bytes(), |x| { + &x[self.start..self.start + self.desc.len] + }) + } else { + BorrowedValue::map(self.buffer.obj_bytes(), |x| &x[self.start..]) + } + } + + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + if self.desc.is_contiguous() { + BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { + &mut x[self.start..self.start + self.desc.len] + }) + } else { + BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| &mut x[self.start..]) + } + } + + fn as_contiguous(&self) -> Option> { + self.desc.is_contiguous().then(|| { + BorrowedValue::map(self.buffer.obj_bytes(), |x| { + &x[self.start..self.start + self.desc.len] + }) + }) + } + + fn _as_contiguous_mut(&self) -> Option> { + self.desc.is_contiguous().then(|| { + BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { + &mut x[self.start..self.start + self.desc.len] + }) + }) + } + + fn append_to(&self, buf: &mut Vec) { + if let Some(bytes) = self.as_contiguous() { + buf.extend_from_slice(&bytes); + } else { + buf.reserve(self.desc.len); + let bytes = &*self.buffer.obj_bytes(); + self.desc.for_each_segment(true, |range| { + let start = (range.start + self.start as isize) as usize; + let end = (range.end + self.start as isize) as usize; + buf.extend_from_slice(&bytes[start..end]); + }) + } + } + + fn contiguous_or_collect R>(&self, f: F) -> R { + let borrowed; + let mut collected; + let v = if let Some(bytes) = self.as_contiguous() { + borrowed = bytes; + &*borrowed + } else { + collected = vec![]; + self.append_to(&mut collected); + &collected + }; + f(v) + } + + /// clone data from memoryview + /// keep the shape, convert to contiguous + pub fn to_contiguous(&self, vm: &VirtualMachine) -> PyBuffer { + let mut data = vec![]; + self.append_to(&mut data); + + if self.desc.ndim() == 0 { + return VecBuffer::from(data) + .into_ref(vm) + .into_pybuffer_with_descriptor(self.desc.clone()); + } + + let mut dim_desc = self.desc.dim_desc.clone(); + dim_desc.last_mut().unwrap().1 = self.desc.itemsize as isize; + dim_desc.last_mut().unwrap().2 = 0; + for i in (0..dim_desc.len() - 1).rev() { + dim_desc[i].1 = dim_desc[i + 1].1 * dim_desc[i + 1].0 as isize; + dim_desc[i].2 = 0; + } + + let desc = BufferDescriptor { + len: self.desc.len, + readonly: self.desc.readonly, + itemsize: self.desc.itemsize, + format: self.desc.format.clone(), + dim_desc, + }; + + VecBuffer::from(data) + .into_ref(vm) + .into_pybuffer_with_descriptor(desc) + } +} + +impl Py { + fn setitem_by_slice( + &self, + slice: &PySlice, + src: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.desc.ndim() != 1 { + return Err(vm.new_not_implemented_error("sub-view are not implemented".to_owned())); + } + + let mut dest = self.new_view(); + dest.init_slice(slice, 0, vm)?; + dest.init_len(); + + if self.is(&src) { + return if !is_equiv_structure(&self.desc, &dest.desc) { + Err(vm.new_value_error( + "memoryview assignment: lvalue and rvalue have different structures".to_owned(), + )) + } else { + // assign self[:] to self + Ok(()) + }; + }; + + let src = if let Some(src) = src.downcast_ref::() { + if self.buffer.obj.is(&src.buffer.obj) { + src.to_contiguous(vm) + } else { + AsBuffer::as_buffer(src, vm)? + } + } else { + PyBuffer::try_from_object(vm, src)? + }; + + if !is_equiv_structure(&src.desc, &dest.desc) { + return Err(vm.new_value_error( + "memoryview assignment: lvalue and rvalue have different structures".to_owned(), + )); + } + + let mut bytes_mut = dest.buffer.obj_bytes_mut(); + let src_bytes = src.obj_bytes(); + dest.desc.zip_eq(&src.desc, true, |a_range, b_range| { + let a_range = (a_range.start + dest.start as isize) as usize + ..(a_range.end + dest.start as isize) as usize; + let b_range = b_range.start as usize..b_range.end as usize; + bytes_mut[a_range].copy_from_slice(&src_bytes[b_range]); + false + }); + + Ok(()) + } +} + +#[pyclass(with( + Py, + Hashable, + Comparable, + AsBuffer, + AsMapping, + AsSequence, + Constructor, + Iterable, + Representable +))] +impl PyMemoryView { + #[pymethod] + pub fn release(&self) { + if self.released.compare_exchange(false, true).is_ok() { + self.buffer.release(); + } + } + + #[pygetset] + fn obj(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.buffer.obj.clone()) + } + + #[pygetset] + fn nbytes(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.len) + } + + #[pygetset] + fn readonly(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.readonly) + } + + #[pygetset] + fn itemsize(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.itemsize) + } + + #[pygetset] + fn ndim(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.ndim()) + } + + #[pygetset] + fn shape(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm)?; + Ok(vm.ctx.new_tuple( + self.desc + .dim_desc + .iter() + .map(|(shape, _, _)| shape.to_pyobject(vm)) + .collect(), + )) + } + + #[pygetset] + fn strides(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm)?; + Ok(vm.ctx.new_tuple( + self.desc + .dim_desc + .iter() + .map(|(_, stride, _)| stride.to_pyobject(vm)) + .collect(), + )) + } + + #[pygetset] + fn suboffsets(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm)?; + Ok(vm.ctx.new_tuple( + self.desc + .dim_desc + .iter() + .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) + .collect(), + )) + } + + #[pygetset] + fn format(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm) + .map(|_| PyStr::from(self.desc.format.clone())) + } + + #[pygetset] + fn contiguous(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.is_contiguous()) + } + + #[pygetset] + fn c_contiguous(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.desc.is_contiguous()) + } + + #[pygetset] + fn f_contiguous(&self, vm: &VirtualMachine) -> PyResult { + // TODO: fortain order + self.try_not_released(vm) + .map(|_| self.desc.ndim() <= 1 && self.desc.is_contiguous()) + } + + #[pymethod(magic)] + fn enter(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { + zelf.try_not_released(vm).map(|_| zelf) + } + + #[pymethod(magic)] + fn exit(&self, _args: FuncArgs) { + self.release(); + } + + #[pymethod(magic)] + fn getitem(zelf: PyRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + zelf.try_not_released(vm)?; + if zelf.desc.ndim() == 0 { + // 0-d memoryview can be referenced using mv[...] or mv[()] only + if needle.is(&vm.ctx.ellipsis) { + return Ok(zelf.into()); + } + if let Some(tuple) = needle.payload::() { + if tuple.is_empty() { + return zelf.unpack_single(0, vm); + } + } + return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); + } + + match SubscriptNeedle::try_from_object(vm, needle)? { + SubscriptNeedle::Index(i) => zelf.getitem_by_idx(i, vm), + SubscriptNeedle::Slice(slice) => zelf.getitem_by_slice(&slice, vm), + SubscriptNeedle::MultiIndex(indices) => zelf.getitem_by_multi_idx(&indices, vm), + } + } + + #[pymethod(magic)] + fn delitem(&self, _needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if self.desc.readonly { + return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); + } + Err(vm.new_type_error("cannot delete memory".to_owned())) + } + + #[pymethod(magic)] + fn len(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm)?; + Ok(if self.desc.ndim() == 0 { + 1 + } else { + // shape for dim[0] + self.desc.dim_desc[0].0 + }) + } + + #[pymethod] + fn tobytes(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm)?; + let mut v = vec![]; + self.append_to(&mut v); + Ok(PyBytes::from(v).into_ref(vm)) + } + #[pymethod] fn tolist(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm)?; @@ -714,178 +848,51 @@ impl PyMemoryView { Ok(self.cast_to_1d(format, vm)?.into_ref(vm)) } } +} - fn eq(zelf: &Py, other: &PyObject, vm: &VirtualMachine) -> PyResult { - if zelf.is(other) { - return Ok(true); - } - if zelf.released.load() { - return Ok(false); - } - - if let Some(other) = other.payload::() { - if other.released.load() { - return Ok(false); - } - } - - let other = match PyBuffer::try_from_borrowed_object(vm, other) { - Ok(buf) => buf, - Err(_) => return Ok(false), - }; - - if !is_equiv_shape(&zelf.desc, &other.desc) { - return Ok(false); - } - - let a_itemsize = zelf.desc.itemsize; - let b_itemsize = other.desc.itemsize; - let a_format_spec = &zelf.format_spec; - let b_format_spec = &Self::parse_format(&other.desc.format, vm)?; - - if zelf.desc.ndim() == 0 { - let a_val = format_unpack(a_format_spec, &zelf.buffer.obj_bytes()[..a_itemsize], vm)?; - let b_val = format_unpack(b_format_spec, &other.obj_bytes()[..b_itemsize], vm)?; - return vm.bool_eq(&a_val, &b_val); - } - - // TODO: optimize cmp by format - let mut ret = Ok(true); - let a_bytes = zelf.buffer.obj_bytes(); - let b_bytes = other.obj_bytes(); - zelf.desc.zip_eq(&other.desc, false, |a_range, b_range| { - let a_range = (a_range.start + zelf.start as isize) as usize - ..(a_range.end + zelf.start as isize) as usize; - let b_range = b_range.start as usize..b_range.end as usize; - let a_val = match format_unpack(a_format_spec, &a_bytes[a_range], vm) { - Ok(val) => val, - Err(e) => { - ret = Err(e); - return true; - } - }; - let b_val = match format_unpack(b_format_spec, &b_bytes[b_range], vm) { - Ok(val) => val, - Err(e) => { - ret = Err(e); - return true; - } - }; - ret = vm.bool_eq(&a_val, &b_val); - if let Ok(b) = ret { - !b - } else { - true - } - }); - ret - } - +#[pyclass] +impl Py { #[pymethod(magic)] - fn reduce_ex(zelf: PyRef, _proto: usize, vm: &VirtualMachine) -> PyResult { - Self::reduce(zelf, vm) - } - - #[pymethod(magic)] - fn reduce(_zelf: PyRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot pickle 'memoryview' object".to_owned())) - } - - fn obj_bytes(&self) -> BorrowedValue<[u8]> { - if self.desc.is_contiguous() { - BorrowedValue::map(self.buffer.obj_bytes(), |x| { - &x[self.start..self.start + self.desc.len] - }) - } else { - BorrowedValue::map(self.buffer.obj_bytes(), |x| &x[self.start..]) + fn setitem( + &self, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.try_not_released(vm)?; + if self.desc.readonly { + return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); } - } - - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - if self.desc.is_contiguous() { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { - &mut x[self.start..self.start + self.desc.len] - }) - } else { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| &mut x[self.start..]) + if value.is(&vm.ctx.none) { + return Err(vm.new_type_error("cannot delete memory".to_owned())); } - } - - fn as_contiguous(&self) -> Option> { - self.desc.is_contiguous().then(|| { - BorrowedValue::map(self.buffer.obj_bytes(), |x| { - &x[self.start..self.start + self.desc.len] - }) - }) - } - - fn _as_contiguous_mut(&self) -> Option> { - self.desc.is_contiguous().then(|| { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { - &mut x[self.start..self.start + self.desc.len] - }) - }) - } - - fn append_to(&self, buf: &mut Vec) { - if let Some(bytes) = self.as_contiguous() { - buf.extend_from_slice(&bytes); - } else { - buf.reserve(self.desc.len); - let bytes = &*self.buffer.obj_bytes(); - self.desc.for_each_segment(true, |range| { - let start = (range.start + self.start as isize) as usize; - let end = (range.end + self.start as isize) as usize; - buf.extend_from_slice(&bytes[start..end]); - }) - } - } - - fn contiguous_or_collect R>(&self, f: F) -> R { - let borrowed; - let mut collected; - let v = if let Some(bytes) = self.as_contiguous() { - borrowed = bytes; - &*borrowed - } else { - collected = vec![]; - self.append_to(&mut collected); - &collected - }; - f(v) - } - - /// clone data from memoryview - /// keep the shape, convert to contiguous - pub fn to_contiguous(&self, vm: &VirtualMachine) -> PyBuffer { - let mut data = vec![]; - self.append_to(&mut data); if self.desc.ndim() == 0 { - return VecBuffer::from(data) - .into_ref(vm) - .into_pybuffer_with_descriptor(self.desc.clone()); + // TODO: merge branches when we got conditional if let + if needle.is(&vm.ctx.ellipsis) { + return self.pack_single(0, value, vm); + } else if let Some(tuple) = needle.payload::() { + if tuple.is_empty() { + return self.pack_single(0, value, vm); + } + } + return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); } - - let mut dim_desc = self.desc.dim_desc.clone(); - dim_desc.last_mut().unwrap().1 = self.desc.itemsize as isize; - dim_desc.last_mut().unwrap().2 = 0; - for i in (0..dim_desc.len() - 1).rev() { - dim_desc[i].1 = dim_desc[i + 1].1 * dim_desc[i + 1].0 as isize; - dim_desc[i].2 = 0; + match SubscriptNeedle::try_from_object(vm, needle)? { + SubscriptNeedle::Index(i) => self.setitem_by_idx(i, value, vm), + SubscriptNeedle::Slice(slice) => self.setitem_by_slice(&slice, value, vm), + SubscriptNeedle::MultiIndex(indices) => self.setitem_by_multi_idx(&indices, value, vm), } + } - let desc = BufferDescriptor { - len: self.desc.len, - readonly: self.desc.readonly, - itemsize: self.desc.itemsize, - format: self.desc.format.clone(), - dim_desc, - }; + #[pymethod(magic)] + fn reduce_ex(&self, _proto: usize, vm: &VirtualMachine) -> PyResult { + self.reduce(vm) + } - VecBuffer::from(data) - .into_ref(vm) - .into_pybuffer_with_descriptor(desc) + #[pymethod(magic)] + fn reduce(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'memoryview' object".to_owned())) } } @@ -977,7 +984,7 @@ impl AsMapping for PyMemoryView { ass_subscript: atomic_func!(|mapping, needle, value, vm| { let zelf = PyMemoryView::mapping_downcast(mapping); if let Some(value) = value { - PyMemoryView::setitem(zelf.to_owned(), needle.to_owned(), value, vm) + zelf.setitem(needle.to_owned(), value, vm) } else { Err(vm.new_type_error("cannot delete memory".to_owned())) }