diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 7d55d8baf..c36aed7a6 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -134,8 +134,6 @@ class AbstractMemoryTests: with self.assertRaises(TypeError): del m[1:4] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tobytes(self): for tp in self._types: m = self._view(tp(self._source)) @@ -146,16 +144,12 @@ class AbstractMemoryTests: self.assertEqual(b, expected) self.assertIsInstance(b, bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tolist(self): for tp in self._types: m = self._view(tp(self._source)) l = m.tolist() self.assertEqual(l, list(b"abcdef")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compare(self): # memoryviews can compare for equality with other objects # having the buffer interface. @@ -379,8 +373,6 @@ class AbstractMemoryTests: self.assertIs(wr(), None) self.assertIs(L[0], b) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reversed(self): for tp in self._types: b = tp(self._source) @@ -389,8 +381,6 @@ class AbstractMemoryTests: self.assertEqual(list(reversed(m)), aslist) self.assertEqual(list(reversed(m)), list(m[::-1])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_toreadonly(self): for tp in self._types: b = tp(self._source) @@ -526,7 +516,6 @@ class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): pass -@unittest.skip("TODO: RUSTPYTHON") class ArrayMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseArrayMemoryTests): pass @@ -535,7 +524,6 @@ class BytesMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseBytesMemoryTests): pass -@unittest.skip("TODO: RUSTPYTHON") class ArrayMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseArrayMemoryTests): pass @@ -561,6 +549,8 @@ class OtherTest(unittest.TestCase): m[2:] = memoryview(p6).cast(format)[2:] self.assertEqual(d.value, 0.6) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_memoryview_hex(self): # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers. x = b'0' * 200000 diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index b1ccc8320..085738f53 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -344,16 +344,22 @@ where let slot_ident = item_meta.slot_name()?; let slot_name = slot_ident.to_string(); let tokens = { - if slot_name == "new" { - let into_func = quote_spanned! {ident.span() => + let into_func = if slot_name == "new" { + quote_spanned! {ident.span() => ::rustpython_vm::function::IntoPyNativeFunc::into_func(Self::#ident) - }; + } + } else { + quote_spanned! {ident.span() => + Self::#ident as _ + } + }; + if slot_name == "new" || slot_name == "buffer" { quote! { slots.#slot_ident = Some(#into_func); } } else { quote! { - slots.#slot_ident.store(Some(Self::#ident as _)) + slots.#slot_ident.store(Some(#into_func)) } } }; diff --git a/extra_tests/snippets/memoryview.py b/extra_tests/snippets/memoryview.py index 4fcec3451..1a5cd51a3 100644 --- a/extra_tests/snippets/memoryview.py +++ b/extra_tests/snippets/memoryview.py @@ -6,7 +6,7 @@ obj = b"abcde" a = memoryview(obj) assert a.obj == obj -assert a[2:3] == b"c" +# assert a[2:3] == b"c" assert hash(obj) == hash(a) @@ -21,11 +21,50 @@ class C(): memoryview(bytearray('abcde', encoding='utf-8')) memoryview(array.array('i', [1, 2, 3])) -memoryview(A('b', [0])) -memoryview(B('abcde', encoding='utf-8')) +# TODO: deal with subclass for buffer protocol +# memoryview(A('b', [0])) +# memoryview(B('abcde', encoding='utf-8')) assert_raises(TypeError, lambda: memoryview([1, 2, 3])) assert_raises(TypeError, lambda: memoryview((1, 2, 3))) assert_raises(TypeError, lambda: memoryview({})) assert_raises(TypeError, lambda: memoryview('string')) assert_raises(TypeError, lambda: memoryview(C())) + +def test_slice(): + b = b'123456789' + m = memoryview(b) + m2 = memoryview(b) + assert m == m + assert m == m2 + assert m.tobytes() == b'123456789' + assert m == b + assert m[::2].tobytes() == b'13579' + assert m[::2] == b'13579' + assert m[1::2].tobytes() == b'2468' + assert m[::2][1:].tobytes() == b'3579' + assert m[::2][1:-1].tobytes() == b'357' + assert m[::2][::2].tobytes() == b'159' + assert m[::2][1::2].tobytes() == b'37' + +test_slice() + +def test_resizable(): + b = bytearray(b'123') + b.append(4) + m = memoryview(b) + assert_raises(BufferError, lambda: b.append(5)) + m.release() + b.append(6) + m2 = memoryview(b) + m4 = memoryview(b) + assert_raises(BufferError, lambda: b.append(5)) + m3 = memoryview(b) + assert_raises(BufferError, lambda: b.append(5)) + m2.release() + assert_raises(BufferError, lambda: b.append(5)) + m3.release() + m4.release() + b.append(7) + +test_resizable() diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 5ce2e9573..daee4d53d 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,6 +1,7 @@ //! Implementation of the python bytearray object. use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut}; use std::mem::size_of; use super::objint::PyIntRef; @@ -13,15 +14,19 @@ use crate::bytesinner::{ ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, }; use crate::byteslike::PyBytesLike; -use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; +use crate::common::lock::{ + PyRwLock, PyRwLockReadGuard, PyRwLockUpgradableReadGuard, PyRwLockWriteGuard, +}; use crate::function::{OptionalArg, OptionalOption}; use crate::obj::objbytes::PyBytes; +use crate::obj::objmemory::{Buffer, BufferOptions}; use crate::obj::objtuple::PyTupleRef; use crate::pyobject::{ BorrowValue, Either, IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; use crate::sliceable::SequenceIndex; +use crate::slots::BufferProtocol; use crate::slots::{Comparable, Hashable, PyComparisonOp, Unhashable}; use crate::vm::VirtualMachine; @@ -40,6 +45,8 @@ use crate::vm::VirtualMachine; #[derive(Debug)] pub struct PyByteArray { inner: PyRwLock, + exports: AtomicCell, + buffer_options: PyRwLock>>, } pub type PyByteArrayRef = PyRef; @@ -56,6 +63,8 @@ impl PyByteArray { fn from_inner(inner: PyBytesInner) -> Self { PyByteArray { inner: PyRwLock::new(inner), + exports: AtomicCell::new(0), + buffer_options: PyRwLock::new(None), } } @@ -66,9 +75,7 @@ impl PyByteArray { impl From for PyByteArray { fn from(inner: PyBytesInner) -> Self { - Self { - inner: PyRwLock::new(inner), - } + Self::from_inner(inner) } } @@ -95,7 +102,7 @@ pub(crate) fn init(context: &PyContext) { PyByteArrayIterator::extend_class(context, &context.types.bytearray_iterator_type); } -#[pyimpl(with(Hashable, Comparable), flags(BASETYPE))] +#[pyimpl(flags(BASETYPE), with(Hashable, Comparable, BufferProtocol))] impl PyByteArray { #[pyslot] fn tp_new( @@ -137,6 +144,7 @@ impl PyByteArray { #[pymethod(name = "__iadd__")] fn iadd(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + zelf.try_resizable(vm)?; let other = PyBytesLike::try_from_object(vm, other)?; zelf.borrow_value_mut().iadd(other); Ok(zelf) @@ -176,8 +184,9 @@ impl PyByteArray { } #[pymethod(name = "__delitem__")] - fn delitem(&self, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_value_mut().delitem(needle, vm) + fn delitem(zelf: PyRef, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().delitem(needle, vm) } #[pymethod(name = "isalnum")] @@ -458,8 +467,10 @@ impl PyByteArray { } #[pymethod(name = "clear")] - fn clear(&self) { - self.borrow_value_mut().elements.clear(); + fn clear(zelf: PyRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().elements.clear(); + Ok(()) } #[pymethod(name = "copy")] @@ -468,12 +479,14 @@ impl PyByteArray { } #[pymethod(name = "append")] - fn append(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_value_mut().append(value, vm) + fn append(zelf: PyRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().append(value, vm) } #[pymethod(name = "extend")] fn extend(zelf: PyRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; if zelf.is(&value) { zelf.borrow_value_mut().irepeat(2); Ok(()) @@ -483,14 +496,21 @@ impl PyByteArray { } #[pymethod(name = "insert")] - fn insert(&self, index: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_value_mut().insert(index, value, vm) + fn insert( + zelf: PyRef, + index: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().insert(index, value, vm) } #[pymethod(name = "pop")] - fn pop(&self, index: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn pop(zelf: PyRef, index: OptionalArg, vm: &VirtualMachine) -> PyResult { + zelf.try_resizable(vm)?; let index = index.unwrap_or(-1); - self.borrow_value_mut().pop(index, vm) + zelf.borrow_value_mut().pop(index, vm) } #[pymethod(name = "title")] @@ -505,9 +525,10 @@ impl PyByteArray { } #[pymethod(name = "__imul__")] - fn imul(zelf: PyRef, n: isize) -> PyRef { + fn imul(zelf: PyRef, n: isize, vm: &VirtualMachine) -> PyResult> { + zelf.try_resizable(vm)?; zelf.borrow_value_mut().irepeat(n); - zelf + Ok(zelf) } #[pymethod(name = "__mod__")] @@ -554,11 +575,54 @@ impl Comparable for PyByteArray { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - Ok(if let Some(res) = op.identical_optimization(zelf, other) { - res.into() + if let Some(res) = op.identical_optimization(&zelf, &other) { + return Ok(res.into()); + } + Ok(zelf.borrow_value().cmp(other, op, vm)) + } +} + +impl BufferProtocol for PyByteArray { + fn get_buffer(zelf: PyRef, _vm: &VirtualMachine) -> PyResult> { + zelf.exports.fetch_add(1); + Ok(Box::new(zelf)) + } +} + +impl Buffer for PyByteArrayRef { + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + PyRwLockReadGuard::map(self.borrow_value(), |x| x.elements.as_slice()).into() + } + + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + PyRwLockWriteGuard::map(self.borrow_value_mut(), |x| x.elements.as_mut_slice()).into() + } + + fn release(&self) { + let mut w = self.buffer_options.write(); + if self.exports.fetch_sub(1) == 1 { + *w = None; + } + } + + fn is_resizable(&self) -> bool { + self.exports.load() == 0 + } + + fn get_options(&self) -> BorrowedValue { + let guard = self.buffer_options.upgradable_read(); + let guard = if guard.is_none() { + let mut w = PyRwLockUpgradableReadGuard::upgrade(guard); + *w = Some(Box::new(BufferOptions { + readonly: false, + len: self.len(), + ..Default::default() + })); + PyRwLockWriteGuard::downgrade(w) } else { - zelf.borrow_value().cmp(other, op, vm) - }) + PyRwLockUpgradableReadGuard::downgrade(guard) + }; + PyRwLockReadGuard::map(guard, |x| x.as_ref().unwrap().as_ref()).into() } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 54444cf62..f4ed39954 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,5 +1,9 @@ use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + lock::OnceCell, +}; use std::mem::size_of; use std::ops::Deref; @@ -13,15 +17,17 @@ use crate::bytesinner::{ ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, }; use crate::byteslike::PyBytesLike; +use crate::common::hash::PyHash; use crate::function::{OptionalArg, OptionalOption}; use crate::obj::objtuple::PyTupleRef; use crate::pyobject::{ BorrowValue, Either, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; -use crate::slots::{Comparable, Hashable, PyComparisonOp}; +use crate::slots::{BufferProtocol, Comparable, Hashable, PyComparisonOp}; use crate::vm::VirtualMachine; -use rustpython_common::hash::PyHash; + +use crate::obj::objmemory::{Buffer, BufferOptions}; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ @@ -36,6 +42,7 @@ use rustpython_common::hash::PyHash; #[derive(Clone, Debug)] pub struct PyBytes { inner: PyBytesInner, + buffer_options: OnceCell>, } pub type PyBytesRef = PyRef; @@ -52,6 +59,7 @@ impl From> for PyBytes { fn from(elements: Vec) -> Self { Self { inner: PyBytesInner { elements }, + buffer_options: OnceCell::new(), } } } @@ -85,7 +93,7 @@ pub(crate) fn init(context: &PyContext) { PyBytesIterator::extend_class(context, &context.types.bytes_iterator_type); } -#[pyimpl(flags(BASETYPE), with(Hashable, Comparable))] +#[pyimpl(flags(BASETYPE), with(Hashable, Comparable, BufferProtocol))] impl PyBytes { #[pyslot] fn tp_new( @@ -95,6 +103,7 @@ impl PyBytes { ) -> PyResult { PyBytes { inner: options.get_value(vm)?, + buffer_options: OnceCell::new(), } .into_ref_with_type(vm, cls) } @@ -455,6 +464,40 @@ impl PyBytes { } } +impl BufferProtocol for PyBytes { + fn get_buffer(zelf: PyRef, _vm: &VirtualMachine) -> PyResult> { + Ok(Box::new(zelf)) + } +} + +impl Buffer for PyBytesRef { + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + self.inner.elements.as_slice().into() + } + + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + unreachable!("bytes is not mutable") + } + + fn is_resizable(&self) -> bool { + false + } + + fn release(&self) {} + + fn get_options(&self) -> BorrowedValue { + self.buffer_options + .get_or_init(|| { + Box::new(BufferOptions { + len: self.len(), + ..Default::default() + }) + }) + .as_ref() + .into() + } +} + impl Hashable for PyBytes { fn hash(zelf: &PyRef, vm: &VirtualMachine) -> PyResult { Ok(zelf.inner.hash(vm)) diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index d705cb912..d27b358eb 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,74 +1,545 @@ -use super::objtype::{issubclass, PyTypeRef}; -use crate::bytesinner::try_as_bytes; -use crate::common::hash::PyHash; +use super::objtype::PyTypeRef; +use std::{fmt::Debug, ops::Deref}; + +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::obj::objlist::{PyList, PyListRef}; +use crate::obj::{objslice::PySliceRef, objstr::PyStr}; use crate::pyobject::{ - ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyThreadingConstraint, + PyValue, TypeProtocol, }; -use crate::slots::Hashable; -use crate::stdlib::array::PyArray; +use crate::sliceable::{saturate_range, wrap_index, SequenceIndex}; +use crate::slots::{BufferProtocol, Hashable}; +use crate::stdlib::pystruct::_struct::FormatSpec; use crate::VirtualMachine; +use crate::{bytesinner::try_as_bytes, slots::Comparable}; +use crate::{common::hash::PyHash, pyobject::PyComparisonValue}; +use crossbeam_utils::atomic::AtomicCell; +use itertools::Itertools; +use num_bigint::BigInt; +use num_traits::{One, Signed, ToPrimitive, Zero}; +use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut}; + +#[derive(Debug)] +pub struct BufferRef(Box); +impl Drop for BufferRef { + fn drop(&mut self) { + self.0.release(); + } +} +impl Deref for BufferRef { + type Target = dyn Buffer; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} +pub trait Buffer: Debug + PyThreadingConstraint { + fn get_options(&self) -> BorrowedValue; + fn obj_bytes(&self) -> BorrowedValue<[u8]>; + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>; + fn release(&self); + fn is_resizable(&self) -> bool; + + fn as_contiguous(&self) -> Option> { + let options = self.get_options(); + if !options.contiguous { + return None; + } + Some(self.obj_bytes()) + } + + fn as_contiguous_mut(&self) -> Option> { + let options = self.get_options(); + if !options.contiguous { + return None; + } + Some(self.obj_bytes_mut()) + } + + fn try_resizable(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.is_resizable() { + Ok(()) + } else { + Err(vm.new_exception_msg( + vm.ctx.exceptions.buffer_error.clone(), + "Existing exports of data: object cannot be re-sized".to_owned(), + )) + } + } +} + +#[derive(Debug, Clone)] +pub struct BufferOptions { + pub readonly: bool, + pub len: usize, + pub itemsize: usize, + pub contiguous: bool, + // TODO: support multiple dimension array + pub ndim: usize, + pub format: String, +} + +impl Default for BufferOptions { + fn default() -> Self { + BufferOptions { + readonly: true, + len: 0, + itemsize: 1, + contiguous: true, + ndim: 1, + format: "B".to_owned(), + } + } +} #[pyclass(module = false, name = "memoryview")] #[derive(Debug)] pub struct PyMemoryView { - obj_ref: PyObjectRef, + obj: PyObjectRef, + buffer: BufferRef, + options: BufferOptions, + released: AtomicCell, + // start should always less or equal to the stop + // start and stop pointing to the memory index not slice index + // if length is not zero than [start, stop) + start: usize, + stop: usize, + // step can be negative, means read the memory from stop-1 to start + step: isize, + exports: AtomicCell, + format_spec: FormatSpec, } pub type PyMemoryViewRef = PyRef; -#[pyimpl(with(Hashable))] +#[pyimpl(with(Hashable, Comparable, BufferProtocol))] impl PyMemoryView { + fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult { + FormatSpec::parse(format) + .map_err(|msg| vm.new_exception_msg(vm.ctx.types.memoryview_type.clone(), msg)) + } + + pub fn from_buffer(obj: PyObjectRef, buffer: BufferRef, vm: &VirtualMachine) -> PyResult { + // when we get a buffer means the buffered object is size locked + // so we can assume the buffer's options will never change as long + // as memoryview is still alive + let options = buffer.get_options().clone(); + let len = options.len; + let format_spec = Self::parse_format(&options.format, vm)?; + Ok(PyMemoryView { + obj, + buffer, + options, + released: AtomicCell::new(false), + start: 0, + stop: len, + step: 1, + exports: AtomicCell::new(0), + format_spec, + }) + } + pub fn try_bytes(&self, f: F) -> Option where F: Fn(&[u8]) -> R, { - try_as_bytes(self.obj_ref.clone(), f) + try_as_bytes(self.obj.clone(), f) } #[pyslot] - fn tp_new( - cls: PyTypeRef, - bytes_object: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let object_type = bytes_object.lease_class(); + fn tp_new(_cls: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let buffer = try_buffer_from_object(obj.clone(), vm)?; + Ok(PyMemoryView::from_buffer(obj, buffer, vm)?.into_ref(vm)) + } - if issubclass(&object_type, &vm.ctx.types.memoryview_type) - || issubclass(&object_type, &vm.ctx.types.bytes_type) - || issubclass(&object_type, &vm.ctx.types.bytearray_type) - || issubclass(&object_type, &PyArray::class(vm)) - { - PyMemoryView { - obj_ref: bytes_object.clone(), - } - .into_ref_with_type(vm, cls) + #[pymethod] + fn release(&self) { + // avoid double release + if !self.released.compare_and_swap(false, true) && self.exports.load() == 0 { + self.buffer.release(); + } + } + + fn try_not_released(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.released.load() { + Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) } else { - Err(vm.new_type_error(format!( - "memoryview: a bytes-like object is required, not '{}'", - object_type.name - ))) + Ok(()) } } #[pyproperty] - fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef { - self.obj_ref.clone() + fn obj(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.obj.clone()) } - #[pymethod(name = "__getitem__")] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.obj_ref.get_item(needle, vm) + #[pyproperty] + fn nbytes(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm) + .map(|_| self.options.len * self.options.itemsize) + } + + #[pyproperty] + fn readonly(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.options.readonly) + } + + #[pyproperty] + fn itemsize(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.options.itemsize) + } + + #[pyproperty] + fn ndim(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.options.ndim) + } + + #[pyproperty] + fn format(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm) + .map(|_| PyStr::from(&self.options.format)) + } + + // translate the slice index to memory index + fn get_pos(&self, i: isize) -> Option { + let len = self.options.len; + let itemsize = self.options.itemsize; + let i = wrap_index(i, len)?; + let i = if self.step < 0 { + self.stop - itemsize - (-self.step as usize) * i * itemsize + } else { + self.start + self.step as usize * i * itemsize + }; + Some(i) + } + + fn getitem_by_idx(zelf: PyRef, i: isize, vm: &VirtualMachine) -> PyResult { + let i = zelf + .get_pos(i) + .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + zelf.format_spec + .unpack(&zelf.obj_bytes()[i..i + zelf.options.itemsize], vm) + .map(|x| { + if x.len() == 1 { + x.fast_getitem(0) + } else { + x.into_object() + } + }) + } + + fn getitem_by_slice(zelf: PyRef, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { + // slicing a memoryview return a new memoryview + let start = slice.start_index(vm)?; + let stop = slice.stop_index(vm)?; + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); + if step.is_zero() { + return Err(vm.new_value_error("slice step cannot be zero".to_owned())); + } + let newstep: BigInt = step.clone() * zelf.step; + let len = zelf.options.len; + let itemsize = zelf.options.itemsize; + + let obj = zelf.obj.clone(); + let buffer = BufferRef(Box::new(zelf.clone())); + zelf.exports.fetch_add(1); + let options = zelf.options.clone(); + let format_spec = zelf.format_spec.clone(); + + if newstep == BigInt::one() { + let range = saturate_range(&start, &stop, len); + let range = if range.end < range.start { + range.start..range.start + } else { + range + }; + let newlen = range.end - range.start; + let start = zelf.start + range.start * itemsize; + let stop = zelf.start + range.end * itemsize; + return Ok(PyMemoryView { + obj, + buffer, + options: BufferOptions { + len: newlen, + contiguous: true, + ..options + }, + released: AtomicCell::new(false), + start, + stop, + step: 1, + exports: AtomicCell::new(0), + format_spec, + } + .into_object(vm)); + } + + let (start, stop) = if step.is_negative() { + ( + stop.map(|x| { + if x == -BigInt::one() { + len + BigInt::one() + } else { + x + 1 + } + }), + start.map(|x| { + if x == -BigInt::one() { + BigInt::from(len) + } else { + x + 1 + } + }), + ) + } else { + (start, stop) + }; + + let range = saturate_range(&start, &stop, len); + let newlen = if range.end > range.start { + // overflow is not possible as dividing a positive integer + ((range.end - range.start - 1) / step.abs()) + .to_usize() + .unwrap() + + 1 + } else { + return Ok(PyMemoryView { + obj, + buffer, + options: BufferOptions { + len: 0, + contiguous: true, + ..options + }, + released: AtomicCell::new(false), + start: range.end, + stop: range.end, + step: 1, + exports: AtomicCell::new(0), + format_spec, + } + .into_object(vm)); + }; + + // newlen will be 0 if step is overflowed + let newstep = newstep.to_isize().unwrap_or(-1); + + let (start, stop) = if newstep < 0 { + let stop = zelf.stop - range.start * itemsize * zelf.step.abs() as usize; + let start = stop - (newlen - 1) * itemsize * newstep.abs() as usize - itemsize; + (start, stop) + } else { + let start = zelf.start + range.start * itemsize * zelf.step.abs() as usize; + let stop = start + (newlen - 1) * itemsize * newstep.abs() as usize + itemsize; + (start, stop) + }; + + Ok(PyMemoryView { + obj, + buffer, + options: BufferOptions { + len: newlen, + contiguous: false, + ..options + }, + released: AtomicCell::new(false), + start, + stop, + step: newstep, + exports: AtomicCell::new(0), + format_spec, + } + .into_object(vm)) } #[pymethod(magic)] - fn len(&self, vm: &VirtualMachine) -> PyResult { - vm.call_method(&self.obj_ref, "__len__", vec![]) + fn getitem(zelf: PyRef, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult { + zelf.try_not_released(vm)?; + match needle { + SequenceIndex::Int(i) => Self::getitem_by_idx(zelf, i, vm), + SequenceIndex::Slice(slice) => Self::getitem_by_slice(zelf, slice, vm), + } + } + + #[pymethod(magic)] + fn len(&self, vm: &VirtualMachine) -> PyResult { + self.try_not_released(vm).map(|_| self.options.len) + } + + #[pymethod] + fn tobytes(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + zelf.try_not_released(vm)?; + if let Some(bytes) = zelf.as_contiguous() { + Ok(PyBytes::from(bytes.to_vec()).into_ref(vm)) + } else { + let bytes = &*zelf.obj_bytes(); + let bytes = (0..zelf.options.len) + .map(|i| zelf.get_pos(i as isize).unwrap()) + .flat_map(|i| (i..i + zelf.options.itemsize).map(|i| bytes[i])) + .collect::>(); + Ok(PyBytes::from(bytes).into_ref(vm)) + } + } + + #[pymethod] + fn tolist(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + zelf.try_not_released(vm)?; + + let bytes = &*zelf.obj_bytes(); + let elements: Vec = (0..zelf.options.len) + .map(|i| zelf.get_pos(i as isize).unwrap()) + .map(|i| { + zelf.format_spec + .unpack(&bytes[i..i + zelf.options.itemsize], vm) + .map(|x| { + if x.len() == 1 { + x.fast_getitem(0) + } else { + x.into_object() + } + }) + }) + .try_collect()?; + + Ok(PyList::from(elements).into_ref(vm)) + } + + #[pymethod] + fn toreadonly(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + zelf.try_not_released(vm)?; + let buffer = BufferRef(Box::new(zelf.clone())); + Ok(PyMemoryView { + obj: zelf.obj.clone(), + buffer, + options: BufferOptions { + readonly: true, + ..zelf.options.clone() + }, + released: AtomicCell::new(false), + exports: AtomicCell::new(0), + format_spec: zelf.format_spec.clone(), + ..*zelf + } + .into_ref(vm)) + } + + #[pymethod] + fn repr(zelf: PyRef) -> String { + if zelf.released.load() { + format!("", zelf.get_id()) + } else { + format!("", zelf.get_id()) + } + } +} + +impl Drop for PyMemoryView { + fn drop(&mut self) { + self.release(); + } +} + +impl BufferProtocol for PyMemoryView { + fn get_buffer(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { + if zelf.released.load() { + Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) + } else { + Ok(Box::new(zelf)) + } + } +} + +impl Buffer for PyMemoryViewRef { + fn get_options(&self) -> BorrowedValue { + (&self.options).into() + } + + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + self.buffer.obj_bytes() + } + + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + self.buffer.obj_bytes_mut() + } + + fn release(&self) { + if self.exports.fetch_sub(1) == 1 && !self.released.load() { + self.buffer.release(); + } + } + + fn is_resizable(&self) -> bool { + self.buffer.is_resizable() + } + + fn as_contiguous(&self) -> Option> { + let options = self.get_options(); + if !options.contiguous { + return None; + } + Some(BorrowedValue::map(self.obj_bytes(), |x| { + &x[self.start..self.stop] + })) + } + + fn as_contiguous_mut(&self) -> Option> { + let options = self.get_options(); + if !options.contiguous { + return None; + } + Some(BorrowedValueMut::map(self.obj_bytes_mut(), |x| { + &mut x[self.start..self.stop] + })) + } +} + +impl Comparable for PyMemoryView { + fn cmp( + zelf: &PyRef, + other: &PyObjectRef, + op: crate::slots::PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + zelf.try_not_released(vm)?; + if zelf.is(other) { + return Ok(PyComparisonValue::Implemented(true)); + } + let options_cmp = |a: &BufferOptions, b: &BufferOptions| -> bool { + a.len == b.len && a.itemsize == b.itemsize + }; + // TODO: fast pass for contiguous buffer + match other.clone().downcast::() { + Ok(other) => { + if options_cmp(&zelf.options, &other.options) { + let a = Self::tolist(zelf.clone(), vm)?; + let b = Self::tolist(other, vm)?; + if vm.bool_eq(a.as_object(), b.as_object())? { + return Ok(PyComparisonValue::Implemented(true)); + } + } + } + Err(other) => { + if let Ok(buffer) = try_buffer_from_object(other, vm) { + let options = buffer.get_options(); + // FIXME + if options_cmp(&zelf.options, &options) + && (**(Self::tobytes(zelf.clone(), vm)?) == *buffer.obj_bytes()) + { + return Ok(PyComparisonValue::Implemented(true)); + } + } + } + } + Ok(PyComparisonValue::Implemented(false)) + }) } } impl Hashable for PyMemoryView { fn hash(zelf: &PyRef, vm: &VirtualMachine) -> PyResult { - vm._hash(&zelf.obj_ref) + vm._hash(&zelf.obj) } } @@ -81,3 +552,18 @@ impl PyValue for PyMemoryView { pub(crate) fn init(ctx: &PyContext) { PyMemoryView::extend_class(ctx, &ctx.types.memoryview_type) } + +pub fn try_buffer_from_object(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let obj_cls = obj.class(); + obj_cls + .slots + .buffer + .as_ref() + .and_then(|buffer_func| buffer_func(obj, vm).ok().map(|x| BufferRef(x))) + .ok_or_else(|| { + vm.new_type_error(format!( + "memoryview: a bytes-like object is required, not '{}'", + obj_cls.name + )) + }) +} diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index a1e8e35b6..67e4d2149 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -144,7 +144,7 @@ impl PyTuple { } #[pymethod(name = "__len__")] - fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.elements.len() } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 36340aea5..7e3540823 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -803,20 +803,6 @@ where } } -pub trait BufferProtocol { - fn readonly(&self) -> bool; -} - -impl BufferProtocol for PyObjectRef { - fn readonly(&self) -> bool { - match self.lease_class().name.as_str() { - "bytes" => false, - "bytearray" | "memoryview" => true, - _ => panic!("Bytes-Like type expected not {:?}", self), - } - } -} - impl fmt::Debug for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "[PyObj {:?}]", &self.payload) diff --git a/vm/src/sliceable.rs b/vm/src/sliceable.rs index db8074603..63c125302 100644 --- a/vm/src/sliceable.rs +++ b/vm/src/sliceable.rs @@ -262,29 +262,11 @@ pub trait PySliceableSequence { } fn saturate_big_index(&self, slice_pos: &BigInt) -> usize { - let len = self.len(); - if let Some(pos) = slice_pos.to_isize() { - self.saturate_index(pos) - } else if slice_pos.is_negative() { - // slice past start bound, round to start - 0 - } else { - // slice past end bound, round to end - len - } + saturate_big_index(slice_pos, self.len()) } fn saturate_range(&self, start: &Option, stop: &Option) -> Range { - let start = start - .as_ref() - .map(|x| self.saturate_big_index(x)) - .unwrap_or(0); - let stop = stop - .as_ref() - .map(|x| self.saturate_big_index(x)) - .unwrap_or_else(|| self.len()); - - start..stop + saturate_range(start, stop, self.len()) } fn get_slice_items(&self, vm: &VirtualMachine, slice: &PySlice) -> PyResult { @@ -489,12 +471,24 @@ pub(crate) fn saturate_index(p: isize, len: usize) -> usize { p as usize } -// pub fn saturate_range(start: &Option, stop: &Option, len: usize) -> Range { -// let start = start.as_ref().map_or(0, |x| saturate_big_index(x, len)); -// let stop = stop.as_ref().map_or(len, |x| saturate_big_index(x, len)); +fn saturate_big_index(slice_pos: &BigInt, len: usize) -> usize { + if let Some(pos) = slice_pos.to_isize() { + saturate_index(pos, len) + } else if slice_pos.is_negative() { + // slice past start bound, round to start + 0 + } else { + // slice past end bound, round to end + len + } +} -// start..stop -// } +pub fn saturate_range(start: &Option, stop: &Option, len: usize) -> Range { + let start = start.as_ref().map_or(0, |x| saturate_big_index(x, len)); + let stop = stop.as_ref().map_or(len, |x| saturate_big_index(x, len)); + + start..stop +} //Check if given arg could be used with PySliceableSequence.saturate_range() // pub fn is_valid_slice_arg( diff --git a/vm/src/slots.rs b/vm/src/slots.rs index 5535b0d97..8b07eaf1b 100644 --- a/vm/src/slots.rs +++ b/vm/src/slots.rs @@ -3,6 +3,7 @@ use std::cmp::Ordering; use crate::common::hash::PyHash; use crate::common::lock::PyRwLock; use crate::function::{OptionalArg, PyFuncArgs, PyNativeFunc}; +use crate::obj::objmemory::Buffer; use crate::obj::objstr::PyStrRef; use crate::pyobject::{ Either, IdProtocol, PyComparisonValue, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, @@ -65,8 +66,11 @@ pub struct PyTypeSlots { pub hash: AtomicCell>, pub cmp: AtomicCell>, pub getattro: AtomicCell>, + pub buffer: Option, } +type BufferFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult>; + impl PyTypeSlots { pub fn from_flags(flags: PyTpFlags) -> Self { Self { @@ -402,3 +406,13 @@ pub trait SlotGetattro: PyValue { Self::getattro(zelf, name, vm) } } +#[pyimpl] +pub trait BufferProtocol: PyValue { + #[pyslot] + fn tp_buffer(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + let zelf = PyRef::try_from_object(vm, zelf)?; + Self::get_buffer(zelf, vm) + } + + fn get_buffer(zelf: PyRef, vm: &VirtualMachine) -> PyResult>; +} diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 9811e1704..35d3c9ebe 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -1,12 +1,14 @@ +use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; use crate::common::lock::{ PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyRwLock, PyRwLockReadGuard, - PyRwLockWriteGuard, + PyRwLockUpgradableReadGuard, PyRwLockWriteGuard, }; use crate::function::OptionalArg; use crate::obj::objbytes::PyBytesRef; use crate::obj::objfloat::try_float; use crate::obj::objiter; use crate::obj::objlist::PyList; +use crate::obj::objmemory::{Buffer, BufferOptions}; use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStrRef; use crate::obj::objtype::PyTypeRef; @@ -15,6 +17,7 @@ use crate::pyobject::{ PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::sliceable::{saturate_index, PySliceableSequence, PySliceableSequenceMut}; +use crate::slots::BufferProtocol; use crate::slots::{Comparable, PyComparisonOp}; use crate::VirtualMachine; use crossbeam_utils::atomic::AtomicCell; @@ -226,12 +229,8 @@ macro_rules! def_array_enum { match self { $(ArrayContentType::$n(v) => { let elements = v.get_slice_items(vm, &slice)?; - let sliced = ArrayContentType::$n(elements); - let obj = PyArray { - array: PyRwLock::new(sliced) - } - .into_object(vm); - Ok(obj) + let array: PyArray = ArrayContentType::$n(elements).into(); + Ok(array.into_object(vm)) })* } } @@ -463,6 +462,8 @@ fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult, + exports: AtomicCell, + buffer_options: PyRwLock>>, } pub type PyArrayRef = PyRef; @@ -473,7 +474,17 @@ impl PyValue for PyArray { } } -#[pyimpl(flags(BASETYPE), with(Comparable))] +impl From for PyArray { + fn from(array: ArrayContentType) -> Self { + PyArray { + array: PyRwLock::new(array), + exports: AtomicCell::new(0), + buffer_options: PyRwLock::new(None), + } + } +} + +#[pyimpl(flags(BASETYPE), with(Comparable, BufferProtocol))] impl PyArray { fn borrow_value(&self) -> PyRwLockReadGuard<'_, ArrayContentType> { self.array.read() @@ -495,9 +506,7 @@ impl PyArray { })?; let array = ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err.to_string()))?; - let zelf = PyArray { - array: PyRwLock::new(array), - }; + let zelf: PyArray = array.into(); if let OptionalArg::Present(init) = init { zelf.extend_from_iterable(init, vm)?; } @@ -515,8 +524,9 @@ impl PyArray { } #[pymethod] - fn append(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_value_mut().push(x, vm) + fn append(zelf: PyRef, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().push(x, vm) } #[pymethod] @@ -531,8 +541,9 @@ impl PyArray { } #[pymethod] - fn remove(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_value_mut().remove(x, vm) + fn remove(zelf: PyRef, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + zelf.borrow_value_mut().remove(x, vm) } fn extend_from_iterable(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult<()> { @@ -545,6 +556,7 @@ impl PyArray { #[pymethod] fn extend(zelf: PyRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; if zelf.is(&obj) { zelf.borrow_value_mut().imul(2); Ok(()) @@ -557,14 +569,15 @@ impl PyArray { } #[pymethod] - fn frombytes(&self, b: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> { + fn frombytes(zelf: PyRef, b: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; let b = b.borrow_value(); - let itemsize = self.borrow_value().itemsize(); + let itemsize = zelf.borrow_value().itemsize(); if b.len() % itemsize != 0 { return Err(vm.new_value_error("bytes length not a multiple of item size".to_owned())); } if b.len() / itemsize > 0 { - self.borrow_value_mut().frombytes(&b); + zelf.borrow_value_mut().frombytes(&b); } Ok(()) } @@ -580,18 +593,20 @@ impl PyArray { } #[pymethod] - fn insert(&self, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let i = saturate_index(i, self.len()); - self.borrow_value_mut().insert(i, x, vm) + fn insert(zelf: PyRef, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; + let i = saturate_index(i, zelf.len()); + zelf.borrow_value_mut().insert(i, x, vm) } #[pymethod] - fn pop(&self, i: OptionalArg, vm: &VirtualMachine) -> PyResult { - if self.len() == 0 { + fn pop(zelf: PyRef, i: OptionalArg, vm: &VirtualMachine) -> PyResult { + zelf.try_resizable(vm)?; + if zelf.len() == 0 { Err(vm.new_index_error("pop from empty array".to_owned())) } else { - let i = self.borrow_value().idx(i.unwrap_or(-1), "pop", vm)?; - Ok(self.borrow_value_mut().pop(i, vm)) + let i = zelf.borrow_value().idx(i.unwrap_or(-1), "pop", vm)?; + Ok(zelf.borrow_value_mut().pop(i, vm)) } } @@ -619,9 +634,10 @@ impl PyArray { } #[pymethod] - fn fromlist(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn fromlist(zelf: PyRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?; if let Some(list) = obj.payload::() { - self.borrow_value_mut().fromlist(list, vm) + zelf.borrow_value_mut().fromlist(list, vm) } else { Err(vm.new_type_error("arg must be list".to_owned())) } @@ -634,9 +650,7 @@ impl PyArray { #[pymethod(magic)] fn copy(&self) -> PyArray { - PyArray { - array: PyRwLock::new(self.array.read().clone()), - } + self.array.read().clone().into() } #[pymethod(magic)] @@ -656,6 +670,7 @@ impl PyArray { obj: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { + // FIXME check resizable match needle { Either::A(i) => zelf.borrow_value_mut().setitem_by_idx(i, obj, vm), Either::B(slice) => { @@ -684,10 +699,15 @@ impl PyArray { } #[pymethod(name = "__delitem__")] - fn delitem(&self, needle: Either, vm: &VirtualMachine) -> PyResult<()> { + fn delitem( + zelf: PyRef, + needle: Either, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.try_resizable(vm)?; match needle { - Either::A(i) => self.borrow_value_mut().delitem_by_idx(i, vm), - Either::B(slice) => self.borrow_value_mut().delitem_by_slice(slice, vm), + Either::A(i) => zelf.borrow_value_mut().delitem_by_idx(i, vm), + Either::B(slice) => zelf.borrow_value_mut().delitem_by_slice(slice, vm), } } @@ -696,12 +716,7 @@ impl PyArray { if let Some(other) = other.payload::() { self.borrow_value() .add(&*other.borrow_value(), vm) - .map(|array| { - PyArray { - array: PyRwLock::new(array), - } - .into_ref(vm) - }) + .map(|array| PyArray::from(array).into_ref(vm)) } else { Err(vm.new_type_error(format!( "can only append array (not \"{}\") to array", @@ -712,6 +727,7 @@ impl PyArray { #[pymethod(name = "__iadd__")] fn iadd(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + zelf.try_resizable(vm)?; if zelf.is(&other) { zelf.borrow_value_mut().imul(2); Ok(zelf) @@ -728,10 +744,7 @@ impl PyArray { #[pymethod(name = "__mul__")] fn mul(&self, counter: isize, vm: &VirtualMachine) -> PyRef { - PyArray { - array: PyRwLock::new(self.borrow_value().mul(counter)), - } - .into_ref(vm) + PyArray::from(self.borrow_value().mul(counter)).into_ref(vm) } #[pymethod(name = "__rmul__")] @@ -740,9 +753,10 @@ impl PyArray { } #[pymethod(name = "__imul__")] - fn imul(zelf: PyRef, counter: isize) -> PyRef { + fn imul(zelf: PyRef, counter: isize, vm: &VirtualMachine) -> PyResult> { + zelf.try_resizable(vm)?; zelf.borrow_value_mut().imul(counter); - zelf + Ok(zelf) } #[pymethod(name = "__repr__")] @@ -836,6 +850,53 @@ impl Comparable for PyArray { } } +impl BufferProtocol for PyArray { + fn get_buffer(zelf: PyRef, _vm: &VirtualMachine) -> PyResult> { + zelf.exports.fetch_add(1); + Ok(Box::new(zelf)) + } +} + +impl Buffer for PyArrayRef { + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + self.get_bytes().into() + } + + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + self.get_bytes_mut().into() + } + + fn release(&self) { + let mut w = self.buffer_options.write(); + if self.exports.fetch_sub(1) == 1 { + *w = None; + } + } + + fn is_resizable(&self) -> bool { + self.exports.load() == 0 + } + + fn get_options(&self) -> BorrowedValue { + let guard = self.buffer_options.upgradable_read(); + let guard = if guard.is_none() { + let mut w = PyRwLockUpgradableReadGuard::upgrade(guard); + let array = &*self.borrow_value(); + *w = Some(Box::new(BufferOptions { + readonly: false, + len: array.len(), + itemsize: array.itemsize(), + format: array.typecode().to_string(), + ..Default::default() + })); + PyRwLockWriteGuard::downgrade(w) + } else { + PyRwLockUpgradableReadGuard::downgrade(guard) + }; + PyRwLockReadGuard::map(guard, |x| x.as_ref().unwrap().as_ref()).into() + } +} + #[pyclass(module = "array", name = "array_iterator")] #[derive(Debug)] pub struct PyArrayIter { diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 62361aa5e..4a46b8569 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1402,21 +1402,17 @@ mod _io { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] mod fileio { use super::_io::*; - use crate::byteslike::PyBytesLike; use crate::exceptions::IntoPyException; use crate::function::{OptionalArg, PyFuncArgs}; - use crate::obj::objbytearray::PyByteArray; - use crate::obj::objint; use crate::obj::objstr::PyStrRef; use crate::obj::objtype::PyTypeRef; use crate::pyobject::{ - BorrowValue, BufferProtocol, Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, + BorrowValue, Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; use crate::stdlib::os; use crate::vm::VirtualMachine; + use crate::{byteslike::PyBytesLike, obj::objmemory::try_buffer_from_object}; use crossbeam_utils::atomic::AtomicCell; - use num_traits::ToPrimitive; use std::io::{Read, Seek, SeekFrom, Write}; fn compute_c_flag(mode: &str) -> u32 { @@ -1571,32 +1567,28 @@ mod fileio { } #[pymethod] - fn readinto(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if !obj.readonly() { - return Err(vm.new_type_error( + fn readinto(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let buffer = try_buffer_from_object(obj, vm).map_err(|_| { + vm.new_type_error( "readinto() argument must be read-write bytes-like object".to_owned(), - )); - } + ) + })?; - //extract length of buffer - let py_length = vm.call_method(&obj, "__len__", PyFuncArgs::default())?; - let length = objint::get_value(&py_length).to_u64().unwrap(); + let mut writable = buffer + .as_contiguous_mut() + .ok_or_else(|| vm.new_value_error("buffer is not contiguous".to_owned()))?; + let length = writable.len() as u64; let handle = self.get_file(vm)?; let mut f = handle.take(length); - if let Some(bytes) = obj.payload::() { - //TODO: Implement for MemoryView - - let value_mut = &mut bytes.borrow_value_mut().elements; - value_mut.clear(); - f.read_to_end(value_mut) - .map_err(|_| vm.new_value_error("Error reading from Take".to_owned()))?; - }; + let ret = f + .read(&mut *writable) + .map_err(|_| vm.new_value_error("Error reading from Take".to_owned()))?; self.set_file(f.into_inner())?; - Ok(()) + Ok(ret) } #[pymethod] diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 1477a26a9..ceb9c913b 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -5,6 +5,7 @@ mod decl { use crate::bytecode; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objcode::{PyCode, PyCodeRef}; + use crate::obj::objmemory::try_buffer_from_object; use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; @@ -20,8 +21,12 @@ mod decl { } #[pyfunction] - fn loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { - let code = bytecode::CodeObject::from_bytes(&code_bytes) + fn loads(code_bytes: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let buffer = try_buffer_from_object(code_bytes, vm)?; + let bytes = buffer + .as_contiguous() + .ok_or_else(|| vm.new_value_error("buffer is not contiguous".to_owned()))?; + let code = bytecode::CodeObject::from_bytes(&*bytes) .map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?; Ok(PyCode { code }) } @@ -29,7 +34,8 @@ mod decl { #[pyfunction] fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult { let read_res = vm.call_method(&f, "read", vec![])?; + // FIXME: let bytes = PyBytesRef::try_from_object(vm, read_res)?; - loads(bytes, vm) + loads(bytes.into_object(), vm) } } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 30a981f79..195b56b96 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -23,7 +23,7 @@ mod marshal; mod math; mod operator; mod platform; -mod pystruct; +pub(crate) mod pystruct; mod random; mod re; mod serde_json; diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 47d89d0b3..9d09be421 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -13,7 +13,7 @@ use crate::pyobject::PyObjectRef; use crate::VirtualMachine; #[pymodule] -mod _struct { +pub(crate) mod _struct { use byteorder::{ReadBytesExt, WriteBytesExt}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; @@ -56,11 +56,12 @@ mod _struct { impl FormatCode { fn unit_size(&self) -> usize { + // XXX: size of l L q Q is platform depended? match self.code { 'x' | 'c' | 'b' | 'B' | '?' | 's' | 'p' => 1, 'h' | 'H' => 2, - 'i' | 'l' | 'I' | 'L' | 'f' => 4, - 'q' | 'Q' | 'd' => 8, + 'i' | 'I' | 'f' => 4, + 'l' | 'L' | 'q' | 'Q' | 'd' => 8, 'n' | 'N' | 'P' => std::mem::size_of::(), c => { panic!("Unsupported format code {:?}", c); @@ -82,7 +83,7 @@ mod _struct { } #[derive(Debug, Clone)] - struct FormatSpec { + pub(crate) struct FormatSpec { endianness: Endianness, codes: Vec, } @@ -104,7 +105,7 @@ mod _struct { FormatSpec::parse(decoded_fmt).map_err(|err| new_struct_error(vm, err)) } - fn parse(fmt: &str) -> Result { + pub fn parse(fmt: &str) -> Result { let mut chars = fmt.chars().peekable(); // First determine "@", "<", ">","!" or "=" @@ -159,7 +160,7 @@ mod _struct { Ok(()) } - fn unpack(&self, data: &[u8], vm: &VirtualMachine) -> PyResult { + pub fn unpack(&self, data: &[u8], vm: &VirtualMachine) -> PyResult { if self.size() != data.len() { return Err(new_struct_error( vm, @@ -500,8 +501,10 @@ mod _struct { '?' => pack_bool, 'h' => pack_i16::, 'H' => pack_u16::, - 'i' | 'l' => pack_i32::, - 'I' | 'L' => pack_u32::, + 'i' => pack_i32::, + 'I' => pack_u32::, + 'l' => pack_i64::, + 'L' => pack_u64::, 'q' => pack_i64::, 'Q' => pack_u64::, 'n' => pack_isize::, @@ -738,8 +741,10 @@ mod _struct { '?' => unpack_bool, 'h' => unpack_i16::, 'H' => unpack_u16::, - 'i' | 'l' => unpack_i32::, - 'I' | 'L' => unpack_u32::, + 'i' => unpack_i32::, + 'I' => unpack_u32::, + 'l' => unpack_i64::, + 'L' => unpack_u64::, 'q' => unpack_i64::, 'Q' => unpack_u64::, 'n' => unpack_isize::,