Implement Buffer Protocol

Lib/test/test_memoryview.py

@@ -134,8 +134,6 @@ class AbstractMemoryTests:
             with self.assertRaises(TypeError):
                 del m[1:4]
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_tobytes(self):
         for tp in self._types:
             m = self._view(tp(self._source))
@@ -146,16 +144,12 @@ class AbstractMemoryTests:
             self.assertEqual(b, expected)
             self.assertIsInstance(b, bytes)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_tolist(self):
         for tp in self._types:
             m = self._view(tp(self._source))
             l = m.tolist()
             self.assertEqual(l, list(b"abcdef"))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_compare(self):
         # memoryviews can compare for equality with other objects
         # having the buffer interface.
@@ -379,8 +373,6 @@ class AbstractMemoryTests:
             self.assertIs(wr(), None)
             self.assertIs(L[0], b)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_reversed(self):
         for tp in self._types:
             b = tp(self._source)
@@ -389,8 +381,6 @@ class AbstractMemoryTests:
             self.assertEqual(list(reversed(m)), aslist)
             self.assertEqual(list(reversed(m)), list(m[::-1]))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_toreadonly(self):
         for tp in self._types:
             b = tp(self._source)
@@ -526,7 +516,6 @@ class BytesMemorySliceTest(unittest.TestCase,
     BaseMemorySliceTests, BaseBytesMemoryTests):
     pass
 
-@unittest.skip("TODO: RUSTPYTHON")
 class ArrayMemorySliceTest(unittest.TestCase,
     BaseMemorySliceTests, BaseArrayMemoryTests):
     pass
@@ -535,7 +524,6 @@ class BytesMemorySliceSliceTest(unittest.TestCase,
     BaseMemorySliceSliceTests, BaseBytesMemoryTests):
     pass
 
-@unittest.skip("TODO: RUSTPYTHON")
 class ArrayMemorySliceSliceTest(unittest.TestCase,
     BaseMemorySliceSliceTests, BaseArrayMemoryTests):
     pass
@@ -561,6 +549,8 @@ class OtherTest(unittest.TestCase):
                 m[2:] = memoryview(p6).cast(format)[2:]
                 self.assertEqual(d.value, 0.6)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_memoryview_hex(self):
         # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers.
         x = b'0' * 200000

derive/src/pyclass.rs

@@ -344,16 +344,22 @@ where
         let slot_ident = item_meta.slot_name()?;
         let slot_name = slot_ident.to_string();
         let tokens = {
-            if slot_name == "new" {
-                let into_func = quote_spanned! {ident.span() =>
+            let into_func = if slot_name == "new" {
+                quote_spanned! {ident.span() =>
                     ::rustpython_vm::function::IntoPyNativeFunc::into_func(Self::#ident)
-                };
+                }
+            } else {
+                quote_spanned! {ident.span() =>
+                    Self::#ident as _
+                }
+            };
+            if slot_name == "new" || slot_name == "buffer" {
                 quote! {
                     slots.#slot_ident = Some(#into_func);
                 }
             } else {
                 quote! {
-                    slots.#slot_ident.store(Some(Self::#ident as _))
+                    slots.#slot_ident.store(Some(#into_func))
                 }
             }
         };

extra_tests/snippets/memoryview.py

@@ -6,7 +6,7 @@ obj = b"abcde"
 a = memoryview(obj)
 assert a.obj == obj
 
-assert a[2:3] == b"c"
+# assert a[2:3] == b"c"
 
 assert hash(obj) == hash(a)
 
@@ -21,11 +21,50 @@ class C():
 
 memoryview(bytearray('abcde', encoding='utf-8'))
 memoryview(array.array('i', [1, 2, 3]))
-memoryview(A('b', [0]))
-memoryview(B('abcde', encoding='utf-8'))
+# TODO: deal with subclass for buffer protocol
+# memoryview(A('b', [0]))
+# memoryview(B('abcde', encoding='utf-8'))
 
 assert_raises(TypeError, lambda: memoryview([1, 2, 3]))
 assert_raises(TypeError, lambda: memoryview((1, 2, 3)))
 assert_raises(TypeError, lambda: memoryview({}))
 assert_raises(TypeError, lambda: memoryview('string'))
 assert_raises(TypeError, lambda: memoryview(C()))
+
+def test_slice():
+    b = b'123456789'
+    m = memoryview(b)
+    m2 = memoryview(b)
+    assert m == m
+    assert m == m2
+    assert m.tobytes() == b'123456789'
+    assert m == b
+    assert m[::2].tobytes() == b'13579'
+    assert m[::2] == b'13579'
+    assert m[1::2].tobytes() == b'2468'
+    assert m[::2][1:].tobytes() == b'3579'
+    assert m[::2][1:-1].tobytes() == b'357'
+    assert m[::2][::2].tobytes() == b'159'
+    assert m[::2][1::2].tobytes() == b'37'
+
+test_slice()
+
+def test_resizable():
+    b = bytearray(b'123')
+    b.append(4)
+    m = memoryview(b)
+    assert_raises(BufferError, lambda: b.append(5))
+    m.release()
+    b.append(6)
+    m2 = memoryview(b)
+    m4 = memoryview(b)
+    assert_raises(BufferError, lambda: b.append(5))
+    m3 = memoryview(b)
+    assert_raises(BufferError, lambda: b.append(5))
+    m2.release()
+    assert_raises(BufferError, lambda: b.append(5))
+    m3.release()
+    m4.release()
+    b.append(7)
+
+test_resizable()

vm/src/obj/objbytearray.rs

@@ -1,6 +1,7 @@
 //! Implementation of the python bytearray object.
 use bstr::ByteSlice;
 use crossbeam_utils::atomic::AtomicCell;
+use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut};
 use std::mem::size_of;
 
 use super::objint::PyIntRef;
@@ -13,15 +14,19 @@ use crate::bytesinner::{
     ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
 };
 use crate::byteslike::PyBytesLike;
-use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard};
+use crate::common::lock::{
+    PyRwLock, PyRwLockReadGuard, PyRwLockUpgradableReadGuard, PyRwLockWriteGuard,
+};
 use crate::function::{OptionalArg, OptionalOption};
 use crate::obj::objbytes::PyBytes;
+use crate::obj::objmemory::{Buffer, BufferOptions};
 use crate::obj::objtuple::PyTupleRef;
 use crate::pyobject::{
     BorrowValue, Either, IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext,
     PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
 };
 use crate::sliceable::SequenceIndex;
+use crate::slots::BufferProtocol;
 use crate::slots::{Comparable, Hashable, PyComparisonOp, Unhashable};
 use crate::vm::VirtualMachine;
 
@@ -40,6 +45,8 @@ use crate::vm::VirtualMachine;
 #[derive(Debug)]
 pub struct PyByteArray {
     inner: PyRwLock<PyBytesInner>,
+    exports: AtomicCell<usize>,
+    buffer_options: PyRwLock<Option<Box<BufferOptions>>>,
 }
 
 pub type PyByteArrayRef = PyRef<PyByteArray>;
@@ -56,6 +63,8 @@ impl PyByteArray {
     fn from_inner(inner: PyBytesInner) -> Self {
         PyByteArray {
             inner: PyRwLock::new(inner),
+            exports: AtomicCell::new(0),
+            buffer_options: PyRwLock::new(None),
         }
     }
 
@@ -66,9 +75,7 @@ impl PyByteArray {
 
 impl From<PyBytesInner> for PyByteArray {
     fn from(inner: PyBytesInner) -> Self {
-        Self {
-            inner: PyRwLock::new(inner),
-        }
+        Self::from_inner(inner)
     }
 }
 
@@ -95,7 +102,7 @@ pub(crate) fn init(context: &PyContext) {
     PyByteArrayIterator::extend_class(context, &context.types.bytearray_iterator_type);
 }
 
-#[pyimpl(with(Hashable, Comparable), flags(BASETYPE))]
+#[pyimpl(flags(BASETYPE), with(Hashable, Comparable, BufferProtocol))]
 impl PyByteArray {
     #[pyslot]
     fn tp_new(
@@ -137,6 +144,7 @@ impl PyByteArray {
 
     #[pymethod(name = "__iadd__")]
     fn iadd(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
+        zelf.try_resizable(vm)?;
         let other = PyBytesLike::try_from_object(vm, other)?;
         zelf.borrow_value_mut().iadd(other);
         Ok(zelf)
@@ -176,8 +184,9 @@ impl PyByteArray {
     }
 
     #[pymethod(name = "__delitem__")]
-    fn delitem(&self, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> {
-        self.borrow_value_mut().delitem(needle, vm)
+    fn delitem(zelf: PyRef<Self>, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().delitem(needle, vm)
     }
 
     #[pymethod(name = "isalnum")]
@@ -458,8 +467,10 @@ impl PyByteArray {
     }
 
     #[pymethod(name = "clear")]
-    fn clear(&self) {
-        self.borrow_value_mut().elements.clear();
+    fn clear(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().elements.clear();
+        Ok(())
     }
 
     #[pymethod(name = "copy")]
@@ -468,12 +479,14 @@ impl PyByteArray {
     }
 
     #[pymethod(name = "append")]
-    fn append(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.borrow_value_mut().append(value, vm)
+    fn append(zelf: PyRef<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().append(value, vm)
     }
 
     #[pymethod(name = "extend")]
     fn extend(zelf: PyRef<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
         if zelf.is(&value) {
             zelf.borrow_value_mut().irepeat(2);
             Ok(())
@@ -483,14 +496,21 @@ impl PyByteArray {
     }
 
     #[pymethod(name = "insert")]
-    fn insert(&self, index: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.borrow_value_mut().insert(index, value, vm)
+    fn insert(
+        zelf: PyRef<Self>,
+        index: isize,
+        value: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().insert(index, value, vm)
     }
 
     #[pymethod(name = "pop")]
-    fn pop(&self, index: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<u8> {
+    fn pop(zelf: PyRef<Self>, index: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<u8> {
+        zelf.try_resizable(vm)?;
         let index = index.unwrap_or(-1);
-        self.borrow_value_mut().pop(index, vm)
+        zelf.borrow_value_mut().pop(index, vm)
     }
 
     #[pymethod(name = "title")]
@@ -505,9 +525,10 @@ impl PyByteArray {
     }
 
     #[pymethod(name = "__imul__")]
-    fn imul(zelf: PyRef<Self>, n: isize) -> PyRef<Self> {
+    fn imul(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
+        zelf.try_resizable(vm)?;
         zelf.borrow_value_mut().irepeat(n);
-        zelf
+        Ok(zelf)
     }
 
     #[pymethod(name = "__mod__")]
@@ -554,11 +575,54 @@ impl Comparable for PyByteArray {
         op: PyComparisonOp,
         vm: &VirtualMachine,
     ) -> PyResult<PyComparisonValue> {
-        Ok(if let Some(res) = op.identical_optimization(zelf, other) {
-            res.into()
+        if let Some(res) = op.identical_optimization(&zelf, &other) {
+            return Ok(res.into());
+        }
+        Ok(zelf.borrow_value().cmp(other, op, vm))
+    }
+}
+
+impl BufferProtocol for PyByteArray {
+    fn get_buffer(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>> {
+        zelf.exports.fetch_add(1);
+        Ok(Box::new(zelf))
+    }
+}
+
+impl Buffer for PyByteArrayRef {
+    fn obj_bytes(&self) -> BorrowedValue<[u8]> {
+        PyRwLockReadGuard::map(self.borrow_value(), |x| x.elements.as_slice()).into()
+    }
+
+    fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
+        PyRwLockWriteGuard::map(self.borrow_value_mut(), |x| x.elements.as_mut_slice()).into()
+    }
+
+    fn release(&self) {
+        let mut w = self.buffer_options.write();
+        if self.exports.fetch_sub(1) == 1 {
+            *w = None;
+        }
+    }
+
+    fn is_resizable(&self) -> bool {
+        self.exports.load() == 0
+    }
+
+    fn get_options(&self) -> BorrowedValue<BufferOptions> {
+        let guard = self.buffer_options.upgradable_read();
+        let guard = if guard.is_none() {
+            let mut w = PyRwLockUpgradableReadGuard::upgrade(guard);
+            *w = Some(Box::new(BufferOptions {
+                readonly: false,
+                len: self.len(),
+                ..Default::default()
+            }));
+            PyRwLockWriteGuard::downgrade(w)
         } else {
-            zelf.borrow_value().cmp(other, op, vm)
-        })
+            PyRwLockUpgradableReadGuard::downgrade(guard)
+        };
+        PyRwLockReadGuard::map(guard, |x| x.as_ref().unwrap().as_ref()).into()
     }
 }
 

vm/src/obj/objbytes.rs

@@ -1,5 +1,9 @@
 use bstr::ByteSlice;
 use crossbeam_utils::atomic::AtomicCell;
+use rustpython_common::{
+    borrow::{BorrowedValue, BorrowedValueMut},
+    lock::OnceCell,
+};
 use std::mem::size_of;
 use std::ops::Deref;
 
@@ -13,15 +17,17 @@ use crate::bytesinner::{
     ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
 };
 use crate::byteslike::PyBytesLike;
+use crate::common::hash::PyHash;
 use crate::function::{OptionalArg, OptionalOption};
 use crate::obj::objtuple::PyTupleRef;
 use crate::pyobject::{
     BorrowValue, Either, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyIterable,
     PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
 };
-use crate::slots::{Comparable, Hashable, PyComparisonOp};
+use crate::slots::{BufferProtocol, Comparable, Hashable, PyComparisonOp};
 use crate::vm::VirtualMachine;
-use rustpython_common::hash::PyHash;
+
+use crate::obj::objmemory::{Buffer, BufferOptions};
 
 /// "bytes(iterable_of_ints) -> bytes\n\
 /// bytes(string, encoding[, errors]) -> bytes\n\
@@ -36,6 +42,7 @@ use rustpython_common::hash::PyHash;
 #[derive(Clone, Debug)]
 pub struct PyBytes {
     inner: PyBytesInner,
+    buffer_options: OnceCell<Box<BufferOptions>>,
 }
 
 pub type PyBytesRef = PyRef<PyBytes>;
@@ -52,6 +59,7 @@ impl From<Vec<u8>> for PyBytes {
     fn from(elements: Vec<u8>) -> Self {
         Self {
             inner: PyBytesInner { elements },
+            buffer_options: OnceCell::new(),
         }
     }
 }
@@ -85,7 +93,7 @@ pub(crate) fn init(context: &PyContext) {
     PyBytesIterator::extend_class(context, &context.types.bytes_iterator_type);
 }
 
-#[pyimpl(flags(BASETYPE), with(Hashable, Comparable))]
+#[pyimpl(flags(BASETYPE), with(Hashable, Comparable, BufferProtocol))]
 impl PyBytes {
     #[pyslot]
     fn tp_new(
@@ -95,6 +103,7 @@ impl PyBytes {
     ) -> PyResult<PyBytesRef> {
         PyBytes {
             inner: options.get_value(vm)?,
+            buffer_options: OnceCell::new(),
         }
         .into_ref_with_type(vm, cls)
     }
@@ -455,6 +464,40 @@ impl PyBytes {
     }
 }
 
+impl BufferProtocol for PyBytes {
+    fn get_buffer(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>> {
+        Ok(Box::new(zelf))
+    }
+}
+
+impl Buffer for PyBytesRef {
+    fn obj_bytes(&self) -> BorrowedValue<[u8]> {
+        self.inner.elements.as_slice().into()
+    }
+
+    fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
+        unreachable!("bytes is not mutable")
+    }
+
+    fn is_resizable(&self) -> bool {
+        false
+    }
+
+    fn release(&self) {}
+
+    fn get_options(&self) -> BorrowedValue<BufferOptions> {
+        self.buffer_options
+            .get_or_init(|| {
+                Box::new(BufferOptions {
+                    len: self.len(),
+                    ..Default::default()
+                })
+            })
+            .as_ref()
+            .into()
+    }
+}
+
 impl Hashable for PyBytes {
     fn hash(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyHash> {
         Ok(zelf.inner.hash(vm))

vm/src/obj/objmemory.rs

@@ -1,74 +1,545 @@
-use super::objtype::{issubclass, PyTypeRef};
-use crate::bytesinner::try_as_bytes;
-use crate::common::hash::PyHash;
+use super::objtype::PyTypeRef;
+use std::{fmt::Debug, ops::Deref};
+
+use crate::obj::objbytes::{PyBytes, PyBytesRef};
+use crate::obj::objlist::{PyList, PyListRef};
+use crate::obj::{objslice::PySliceRef, objstr::PyStr};
 use crate::pyobject::{
-    ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol,
+    IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyThreadingConstraint,
+    PyValue, TypeProtocol,
 };
-use crate::slots::Hashable;
-use crate::stdlib::array::PyArray;
+use crate::sliceable::{saturate_range, wrap_index, SequenceIndex};
+use crate::slots::{BufferProtocol, Hashable};
+use crate::stdlib::pystruct::_struct::FormatSpec;
 use crate::VirtualMachine;
+use crate::{bytesinner::try_as_bytes, slots::Comparable};
+use crate::{common::hash::PyHash, pyobject::PyComparisonValue};
+use crossbeam_utils::atomic::AtomicCell;
+use itertools::Itertools;
+use num_bigint::BigInt;
+use num_traits::{One, Signed, ToPrimitive, Zero};
+use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut};
+
+#[derive(Debug)]
+pub struct BufferRef(Box<dyn Buffer>);
+impl Drop for BufferRef {
+    fn drop(&mut self) {
+        self.0.release();
+    }
+}
+impl Deref for BufferRef {
+    type Target = dyn Buffer;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+pub trait Buffer: Debug + PyThreadingConstraint {
+    fn get_options(&self) -> BorrowedValue<BufferOptions>;
+    fn obj_bytes(&self) -> BorrowedValue<[u8]>;
+    fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>;
+    fn release(&self);
+    fn is_resizable(&self) -> bool;
+
+    fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
+        let options = self.get_options();
+        if !options.contiguous {
+            return None;
+        }
+        Some(self.obj_bytes())
+    }
+
+    fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
+        let options = self.get_options();
+        if !options.contiguous {
+            return None;
+        }
+        Some(self.obj_bytes_mut())
+    }
+
+    fn try_resizable(&self, vm: &VirtualMachine) -> PyResult<()> {
+        if self.is_resizable() {
+            Ok(())
+        } else {
+            Err(vm.new_exception_msg(
+                vm.ctx.exceptions.buffer_error.clone(),
+                "Existing exports of data: object cannot be re-sized".to_owned(),
+            ))
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct BufferOptions {
+    pub readonly: bool,
+    pub len: usize,
+    pub itemsize: usize,
+    pub contiguous: bool,
+    // TODO: support multiple dimension array
+    pub ndim: usize,
+    pub format: String,
+}
+
+impl Default for BufferOptions {
+    fn default() -> Self {
+        BufferOptions {
+            readonly: true,
+            len: 0,
+            itemsize: 1,
+            contiguous: true,
+            ndim: 1,
+            format: "B".to_owned(),
+        }
+    }
+}
 
 #[pyclass(module = false, name = "memoryview")]
 #[derive(Debug)]
 pub struct PyMemoryView {
-    obj_ref: PyObjectRef,
+    obj: PyObjectRef,
+    buffer: BufferRef,
+    options: BufferOptions,
+    released: AtomicCell<bool>,
+    // start should always less or equal to the stop
+    // start and stop pointing to the memory index not slice index
+    // if length is not zero than [start, stop)
+    start: usize,
+    stop: usize,
+    // step can be negative, means read the memory from stop-1 to start
+    step: isize,
+    exports: AtomicCell<usize>,
+    format_spec: FormatSpec,
 }
 
 pub type PyMemoryViewRef = PyRef<PyMemoryView>;
 
-#[pyimpl(with(Hashable))]
+#[pyimpl(with(Hashable, Comparable, BufferProtocol))]
 impl PyMemoryView {
+    fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult<FormatSpec> {
+        FormatSpec::parse(format)
+            .map_err(|msg| vm.new_exception_msg(vm.ctx.types.memoryview_type.clone(), msg))
+    }
+
+    pub fn from_buffer(obj: PyObjectRef, buffer: BufferRef, vm: &VirtualMachine) -> PyResult<Self> {
+        // when we get a buffer means the buffered object is size locked
+        // so we can assume the buffer's options will never change as long
+        // as memoryview is still alive
+        let options = buffer.get_options().clone();
+        let len = options.len;
+        let format_spec = Self::parse_format(&options.format, vm)?;
+        Ok(PyMemoryView {
+            obj,
+            buffer,
+            options,
+            released: AtomicCell::new(false),
+            start: 0,
+            stop: len,
+            step: 1,
+            exports: AtomicCell::new(0),
+            format_spec,
+        })
+    }
+
     pub fn try_bytes<F, R>(&self, f: F) -> Option<R>
     where
         F: Fn(&[u8]) -> R,
     {
-        try_as_bytes(self.obj_ref.clone(), f)
+        try_as_bytes(self.obj.clone(), f)
     }
 
     #[pyslot]
-    fn tp_new(
-        cls: PyTypeRef,
-        bytes_object: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyMemoryViewRef> {
-        let object_type = bytes_object.lease_class();
+    fn tp_new(_cls: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyMemoryViewRef> {
+        let buffer = try_buffer_from_object(obj.clone(), vm)?;
+        Ok(PyMemoryView::from_buffer(obj, buffer, vm)?.into_ref(vm))
+    }
 
-        if issubclass(&object_type, &vm.ctx.types.memoryview_type)
-            || issubclass(&object_type, &vm.ctx.types.bytes_type)
-            || issubclass(&object_type, &vm.ctx.types.bytearray_type)
-            || issubclass(&object_type, &PyArray::class(vm))
-        {
-            PyMemoryView {
-                obj_ref: bytes_object.clone(),
-            }
-            .into_ref_with_type(vm, cls)
+    #[pymethod]
+    fn release(&self) {
+        // avoid double release
+        if !self.released.compare_and_swap(false, true) && self.exports.load() == 0 {
+            self.buffer.release();
+        }
+    }
+
+    fn try_not_released(&self, vm: &VirtualMachine) -> PyResult<()> {
+        if self.released.load() {
+            Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned()))
         } else {
-            Err(vm.new_type_error(format!(
-                "memoryview: a bytes-like object is required, not '{}'",
-                object_type.name
-            )))
+            Ok(())
         }
     }
 
     #[pyproperty]
-    fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef {
-        self.obj_ref.clone()
+    fn obj(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
+        self.try_not_released(vm).map(|_| self.obj.clone())
     }
 
-    #[pymethod(name = "__getitem__")]
-    fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult {
-        self.obj_ref.get_item(needle, vm)
+    #[pyproperty]
+    fn nbytes(&self, vm: &VirtualMachine) -> PyResult<usize> {
+        self.try_not_released(vm)
+            .map(|_| self.options.len * self.options.itemsize)
+    }
+
+    #[pyproperty]
+    fn readonly(&self, vm: &VirtualMachine) -> PyResult<bool> {
+        self.try_not_released(vm).map(|_| self.options.readonly)
+    }
+
+    #[pyproperty]
+    fn itemsize(&self, vm: &VirtualMachine) -> PyResult<usize> {
+        self.try_not_released(vm).map(|_| self.options.itemsize)
+    }
+
+    #[pyproperty]
+    fn ndim(&self, vm: &VirtualMachine) -> PyResult<usize> {
+        self.try_not_released(vm).map(|_| self.options.ndim)
+    }
+
+    #[pyproperty]
+    fn format(&self, vm: &VirtualMachine) -> PyResult<PyStr> {
+        self.try_not_released(vm)
+            .map(|_| PyStr::from(&self.options.format))
+    }
+
+    // translate the slice index to memory index
+    fn get_pos(&self, i: isize) -> Option<usize> {
+        let len = self.options.len;
+        let itemsize = self.options.itemsize;
+        let i = wrap_index(i, len)?;
+        let i = if self.step < 0 {
+            self.stop - itemsize - (-self.step as usize) * i * itemsize
+        } else {
+            self.start + self.step as usize * i * itemsize
+        };
+        Some(i)
+    }
+
+    fn getitem_by_idx(zelf: PyRef<Self>, i: isize, vm: &VirtualMachine) -> PyResult {
+        let i = zelf
+            .get_pos(i)
+            .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?;
+        zelf.format_spec
+            .unpack(&zelf.obj_bytes()[i..i + zelf.options.itemsize], vm)
+            .map(|x| {
+                if x.len() == 1 {
+                    x.fast_getitem(0)
+                } else {
+                    x.into_object()
+                }
+            })
+    }
+
+    fn getitem_by_slice(zelf: PyRef<Self>, slice: PySliceRef, vm: &VirtualMachine) -> PyResult {
+        // slicing a memoryview return a new memoryview
+        let start = slice.start_index(vm)?;
+        let stop = slice.stop_index(vm)?;
+        let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one);
+        if step.is_zero() {
+            return Err(vm.new_value_error("slice step cannot be zero".to_owned()));
+        }
+        let newstep: BigInt = step.clone() * zelf.step;
+        let len = zelf.options.len;
+        let itemsize = zelf.options.itemsize;
+
+        let obj = zelf.obj.clone();
+        let buffer = BufferRef(Box::new(zelf.clone()));
+        zelf.exports.fetch_add(1);
+        let options = zelf.options.clone();
+        let format_spec = zelf.format_spec.clone();
+
+        if newstep == BigInt::one() {
+            let range = saturate_range(&start, &stop, len);
+            let range = if range.end < range.start {
+                range.start..range.start
+            } else {
+                range
+            };
+            let newlen = range.end - range.start;
+            let start = zelf.start + range.start * itemsize;
+            let stop = zelf.start + range.end * itemsize;
+            return Ok(PyMemoryView {
+                obj,
+                buffer,
+                options: BufferOptions {
+                    len: newlen,
+                    contiguous: true,
+                    ..options
+                },
+                released: AtomicCell::new(false),
+                start,
+                stop,
+                step: 1,
+                exports: AtomicCell::new(0),
+                format_spec,
+            }
+            .into_object(vm));
+        }
+
+        let (start, stop) = if step.is_negative() {
+            (
+                stop.map(|x| {
+                    if x == -BigInt::one() {
+                        len + BigInt::one()
+                    } else {
+                        x + 1
+                    }
+                }),
+                start.map(|x| {
+                    if x == -BigInt::one() {
+                        BigInt::from(len)
+                    } else {
+                        x + 1
+                    }
+                }),
+            )
+        } else {
+            (start, stop)
+        };
+
+        let range = saturate_range(&start, &stop, len);
+        let newlen = if range.end > range.start {
+            // overflow is not possible as dividing a positive integer
+            ((range.end - range.start - 1) / step.abs())
+                .to_usize()
+                .unwrap()
+                + 1
+        } else {
+            return Ok(PyMemoryView {
+                obj,
+                buffer,
+                options: BufferOptions {
+                    len: 0,
+                    contiguous: true,
+                    ..options
+                },
+                released: AtomicCell::new(false),
+                start: range.end,
+                stop: range.end,
+                step: 1,
+                exports: AtomicCell::new(0),
+                format_spec,
+            }
+            .into_object(vm));
+        };
+
+        // newlen will be 0 if step is overflowed
+        let newstep = newstep.to_isize().unwrap_or(-1);
+
+        let (start, stop) = if newstep < 0 {
+            let stop = zelf.stop - range.start * itemsize * zelf.step.abs() as usize;
+            let start = stop - (newlen - 1) * itemsize * newstep.abs() as usize - itemsize;
+            (start, stop)
+        } else {
+            let start = zelf.start + range.start * itemsize * zelf.step.abs() as usize;
+            let stop = start + (newlen - 1) * itemsize * newstep.abs() as usize + itemsize;
+            (start, stop)
+        };
+
+        Ok(PyMemoryView {
+            obj,
+            buffer,
+            options: BufferOptions {
+                len: newlen,
+                contiguous: false,
+                ..options
+            },
+            released: AtomicCell::new(false),
+            start,
+            stop,
+            step: newstep,
+            exports: AtomicCell::new(0),
+            format_spec,
+        }
+        .into_object(vm))
     }
 
     #[pymethod(magic)]
-    fn len(&self, vm: &VirtualMachine) -> PyResult {
-        vm.call_method(&self.obj_ref, "__len__", vec![])
+    fn getitem(zelf: PyRef<Self>, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult {
+        zelf.try_not_released(vm)?;
+        match needle {
+            SequenceIndex::Int(i) => Self::getitem_by_idx(zelf, i, vm),
+            SequenceIndex::Slice(slice) => Self::getitem_by_slice(zelf, slice, vm),
+        }
+    }
+
+    #[pymethod(magic)]
+    fn len(&self, vm: &VirtualMachine) -> PyResult<usize> {
+        self.try_not_released(vm).map(|_| self.options.len)
+    }
+
+    #[pymethod]
+    fn tobytes(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyBytesRef> {
+        zelf.try_not_released(vm)?;
+        if let Some(bytes) = zelf.as_contiguous() {
+            Ok(PyBytes::from(bytes.to_vec()).into_ref(vm))
+        } else {
+            let bytes = &*zelf.obj_bytes();
+            let bytes = (0..zelf.options.len)
+                .map(|i| zelf.get_pos(i as isize).unwrap())
+                .flat_map(|i| (i..i + zelf.options.itemsize).map(|i| bytes[i]))
+                .collect::<Vec<u8>>();
+            Ok(PyBytes::from(bytes).into_ref(vm))
+        }
+    }
+
+    #[pymethod]
+    fn tolist(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyListRef> {
+        zelf.try_not_released(vm)?;
+
+        let bytes = &*zelf.obj_bytes();
+        let elements: Vec<PyObjectRef> = (0..zelf.options.len)
+            .map(|i| zelf.get_pos(i as isize).unwrap())
+            .map(|i| {
+                zelf.format_spec
+                    .unpack(&bytes[i..i + zelf.options.itemsize], vm)
+                    .map(|x| {
+                        if x.len() == 1 {
+                            x.fast_getitem(0)
+                        } else {
+                            x.into_object()
+                        }
+                    })
+            })
+            .try_collect()?;
+
+        Ok(PyList::from(elements).into_ref(vm))
+    }
+
+    #[pymethod]
+    fn toreadonly(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyMemoryViewRef> {
+        zelf.try_not_released(vm)?;
+        let buffer = BufferRef(Box::new(zelf.clone()));
+        Ok(PyMemoryView {
+            obj: zelf.obj.clone(),
+            buffer,
+            options: BufferOptions {
+                readonly: true,
+                ..zelf.options.clone()
+            },
+            released: AtomicCell::new(false),
+            exports: AtomicCell::new(0),
+            format_spec: zelf.format_spec.clone(),
+            ..*zelf
+        }
+        .into_ref(vm))
+    }
+
+    #[pymethod]
+    fn repr(zelf: PyRef<Self>) -> String {
+        if zelf.released.load() {
+            format!("<released memory at 0x{:x}>", zelf.get_id())
+        } else {
+            format!("<memory at 0x{:x}>", zelf.get_id())
+        }
+    }
+}
+
+impl Drop for PyMemoryView {
+    fn drop(&mut self) {
+        self.release();
+    }
+}
+
+impl BufferProtocol for PyMemoryView {
+    fn get_buffer(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>> {
+        if zelf.released.load() {
+            Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned()))
+        } else {
+            Ok(Box::new(zelf))
+        }
+    }
+}
+
+impl Buffer for PyMemoryViewRef {
+    fn get_options(&self) -> BorrowedValue<BufferOptions> {
+        (&self.options).into()
+    }
+
+    fn obj_bytes(&self) -> BorrowedValue<[u8]> {
+        self.buffer.obj_bytes()
+    }
+
+    fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
+        self.buffer.obj_bytes_mut()
+    }
+
+    fn release(&self) {
+        if self.exports.fetch_sub(1) == 1 && !self.released.load() {
+            self.buffer.release();
+        }
+    }
+
+    fn is_resizable(&self) -> bool {
+        self.buffer.is_resizable()
+    }
+
+    fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
+        let options = self.get_options();
+        if !options.contiguous {
+            return None;
+        }
+        Some(BorrowedValue::map(self.obj_bytes(), |x| {
+            &x[self.start..self.stop]
+        }))
+    }
+
+    fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
+        let options = self.get_options();
+        if !options.contiguous {
+            return None;
+        }
+        Some(BorrowedValueMut::map(self.obj_bytes_mut(), |x| {
+            &mut x[self.start..self.stop]
+        }))
+    }
+}
+
+impl Comparable for PyMemoryView {
+    fn cmp(
+        zelf: &PyRef<Self>,
+        other: &PyObjectRef,
+        op: crate::slots::PyComparisonOp,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyComparisonValue> {
+        op.eq_only(|| {
+            zelf.try_not_released(vm)?;
+            if zelf.is(other) {
+                return Ok(PyComparisonValue::Implemented(true));
+            }
+            let options_cmp = |a: &BufferOptions, b: &BufferOptions| -> bool {
+                a.len == b.len && a.itemsize == b.itemsize
+            };
+            // TODO: fast pass for contiguous buffer
+            match other.clone().downcast::<PyMemoryView>() {
+                Ok(other) => {
+                    if options_cmp(&zelf.options, &other.options) {
+                        let a = Self::tolist(zelf.clone(), vm)?;
+                        let b = Self::tolist(other, vm)?;
+                        if vm.bool_eq(a.as_object(), b.as_object())? {
+                            return Ok(PyComparisonValue::Implemented(true));
+                        }
+                    }
+                }
+                Err(other) => {
+                    if let Ok(buffer) = try_buffer_from_object(other, vm) {
+                        let options = buffer.get_options();
+                        // FIXME
+                        if options_cmp(&zelf.options, &options)
+                            && (**(Self::tobytes(zelf.clone(), vm)?) == *buffer.obj_bytes())
+                        {
+                            return Ok(PyComparisonValue::Implemented(true));
+                        }
+                    }
+                }
+            }
+            Ok(PyComparisonValue::Implemented(false))
+        })
     }
 }
 
 impl Hashable for PyMemoryView {
     fn hash(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyHash> {
-        vm._hash(&zelf.obj_ref)
+        vm._hash(&zelf.obj)
     }
 }
 
@@ -81,3 +552,18 @@ impl PyValue for PyMemoryView {
 pub(crate) fn init(ctx: &PyContext) {
     PyMemoryView::extend_class(ctx, &ctx.types.memoryview_type)
 }
+
+pub fn try_buffer_from_object(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<BufferRef> {
+    let obj_cls = obj.class();
+    obj_cls
+        .slots
+        .buffer
+        .as_ref()
+        .and_then(|buffer_func| buffer_func(obj, vm).ok().map(|x| BufferRef(x)))
+        .ok_or_else(|| {
+            vm.new_type_error(format!(
+                "memoryview: a bytes-like object is required, not '{}'",
+                obj_cls.name
+            ))
+        })
+}

vm/src/obj/objtuple.rs

@@ -144,7 +144,7 @@ impl PyTuple {
     }
 
     #[pymethod(name = "__len__")]
-    fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         self.elements.len()
     }
 

vm/src/pyobject.rs

@@ -803,20 +803,6 @@ where
     }
 }
 
-pub trait BufferProtocol {
-    fn readonly(&self) -> bool;
-}
-
-impl BufferProtocol for PyObjectRef {
-    fn readonly(&self) -> bool {
-        match self.lease_class().name.as_str() {
-            "bytes" => false,
-            "bytearray" | "memoryview" => true,
-            _ => panic!("Bytes-Like type expected not {:?}", self),
-        }
-    }
-}
-
 impl fmt::Debug for PyObject<dyn PyObjectPayload> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "[PyObj {:?}]", &self.payload)

vm/src/sliceable.rs

@@ -262,29 +262,11 @@ pub trait PySliceableSequence {
     }
 
     fn saturate_big_index(&self, slice_pos: &BigInt) -> usize {
-        let len = self.len();
-        if let Some(pos) = slice_pos.to_isize() {
-            self.saturate_index(pos)
-        } else if slice_pos.is_negative() {
-            // slice past start bound, round to start
-            0
-        } else {
-            // slice past end bound, round to end
-            len
-        }
+        saturate_big_index(slice_pos, self.len())
     }
 
     fn saturate_range(&self, start: &Option<BigInt>, stop: &Option<BigInt>) -> Range<usize> {
-        let start = start
-            .as_ref()
-            .map(|x| self.saturate_big_index(x))
-            .unwrap_or(0);
-        let stop = stop
-            .as_ref()
-            .map(|x| self.saturate_big_index(x))
-            .unwrap_or_else(|| self.len());
-
-        start..stop
+        saturate_range(start, stop, self.len())
     }
 
     fn get_slice_items(&self, vm: &VirtualMachine, slice: &PySlice) -> PyResult<Self::Sliced> {
@@ -489,12 +471,24 @@ pub(crate) fn saturate_index(p: isize, len: usize) -> usize {
     p as usize
 }
 
-// pub fn saturate_range(start: &Option<BigInt>, stop: &Option<BigInt>, len: usize) -> Range<usize> {
-//     let start = start.as_ref().map_or(0, |x| saturate_big_index(x, len));
-//     let stop = stop.as_ref().map_or(len, |x| saturate_big_index(x, len));
+fn saturate_big_index(slice_pos: &BigInt, len: usize) -> usize {
+    if let Some(pos) = slice_pos.to_isize() {
+        saturate_index(pos, len)
+    } else if slice_pos.is_negative() {
+        // slice past start bound, round to start
+        0
+    } else {
+        // slice past end bound, round to end
+        len
+    }
+}
 
-//     start..stop
-// }
+pub fn saturate_range(start: &Option<BigInt>, stop: &Option<BigInt>, len: usize) -> Range<usize> {
+    let start = start.as_ref().map_or(0, |x| saturate_big_index(x, len));
+    let stop = stop.as_ref().map_or(len, |x| saturate_big_index(x, len));
+
+    start..stop
+}
 
 //Check if given arg could be used with PySliceableSequence.saturate_range()
 // pub fn is_valid_slice_arg(

vm/src/slots.rs

@@ -3,6 +3,7 @@ use std::cmp::Ordering;
 use crate::common::hash::PyHash;
 use crate::common::lock::PyRwLock;
 use crate::function::{OptionalArg, PyFuncArgs, PyNativeFunc};
+use crate::obj::objmemory::Buffer;
 use crate::obj::objstr::PyStrRef;
 use crate::pyobject::{
     Either, IdProtocol, PyComparisonValue, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
@@ -65,8 +66,11 @@ pub struct PyTypeSlots {
     pub hash: AtomicCell<Option<HashFunc>>,
     pub cmp: AtomicCell<Option<CmpFunc>>,
     pub getattro: AtomicCell<Option<GetattroFunc>>,
+    pub buffer: Option<BufferFunc>,
 }
 
+type BufferFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult<Box<dyn Buffer>>;
+
 impl PyTypeSlots {
     pub fn from_flags(flags: PyTpFlags) -> Self {
         Self {
@@ -402,3 +406,13 @@ pub trait SlotGetattro: PyValue {
         Self::getattro(zelf, name, vm)
     }
 }
+#[pyimpl]
+pub trait BufferProtocol: PyValue {
+    #[pyslot]
+    fn tp_buffer(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>> {
+        let zelf = PyRef::try_from_object(vm, zelf)?;
+        Self::get_buffer(zelf, vm)
+    }
+
+    fn get_buffer(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>>;
+}

vm/src/stdlib/array.rs

@@ -1,12 +1,14 @@
+use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
 use crate::common::lock::{
     PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyRwLock, PyRwLockReadGuard,
-    PyRwLockWriteGuard,
+    PyRwLockUpgradableReadGuard, PyRwLockWriteGuard,
 };
 use crate::function::OptionalArg;
 use crate::obj::objbytes::PyBytesRef;
 use crate::obj::objfloat::try_float;
 use crate::obj::objiter;
 use crate::obj::objlist::PyList;
+use crate::obj::objmemory::{Buffer, BufferOptions};
 use crate::obj::objslice::PySliceRef;
 use crate::obj::objstr::PyStrRef;
 use crate::obj::objtype::PyTypeRef;
@@ -15,6 +17,7 @@ use crate::pyobject::{
     PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol,
 };
 use crate::sliceable::{saturate_index, PySliceableSequence, PySliceableSequenceMut};
+use crate::slots::BufferProtocol;
 use crate::slots::{Comparable, PyComparisonOp};
 use crate::VirtualMachine;
 use crossbeam_utils::atomic::AtomicCell;
@@ -226,12 +229,8 @@ macro_rules! def_array_enum {
                 match self {
                     $(ArrayContentType::$n(v) => {
                         let elements = v.get_slice_items(vm, &slice)?;
-                        let sliced = ArrayContentType::$n(elements);
-                        let obj = PyArray {
-                            array: PyRwLock::new(sliced)
-                        }
-                        .into_object(vm);
-                        Ok(obj)
+                        let array: PyArray = ArrayContentType::$n(elements).into();
+                        Ok(array.into_object(vm))
                     })*
                 }
             }
@@ -463,6 +462,8 @@ fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f
 #[derive(Debug)]
 pub struct PyArray {
     array: PyRwLock<ArrayContentType>,
+    exports: AtomicCell<usize>,
+    buffer_options: PyRwLock<Option<Box<BufferOptions>>>,
 }
 
 pub type PyArrayRef = PyRef<PyArray>;
@@ -473,7 +474,17 @@ impl PyValue for PyArray {
     }
 }
 
-#[pyimpl(flags(BASETYPE), with(Comparable))]
+impl From<ArrayContentType> for PyArray {
+    fn from(array: ArrayContentType) -> Self {
+        PyArray {
+            array: PyRwLock::new(array),
+            exports: AtomicCell::new(0),
+            buffer_options: PyRwLock::new(None),
+        }
+    }
+}
+
+#[pyimpl(flags(BASETYPE), with(Comparable, BufferProtocol))]
 impl PyArray {
     fn borrow_value(&self) -> PyRwLockReadGuard<'_, ArrayContentType> {
         self.array.read()
@@ -495,9 +506,7 @@ impl PyArray {
         })?;
         let array =
             ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err.to_string()))?;
-        let zelf = PyArray {
-            array: PyRwLock::new(array),
-        };
+        let zelf: PyArray = array.into();
         if let OptionalArg::Present(init) = init {
             zelf.extend_from_iterable(init, vm)?;
         }
@@ -515,8 +524,9 @@ impl PyArray {
     }
 
     #[pymethod]
-    fn append(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.borrow_value_mut().push(x, vm)
+    fn append(zelf: PyRef<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().push(x, vm)
     }
 
     #[pymethod]
@@ -531,8 +541,9 @@ impl PyArray {
     }
 
     #[pymethod]
-    fn remove(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.borrow_value_mut().remove(x, vm)
+    fn remove(zelf: PyRef<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        zelf.borrow_value_mut().remove(x, vm)
     }
 
     fn extend_from_iterable(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult<()> {
@@ -545,6 +556,7 @@ impl PyArray {
 
     #[pymethod]
     fn extend(zelf: PyRef<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
         if zelf.is(&obj) {
             zelf.borrow_value_mut().imul(2);
             Ok(())
@@ -557,14 +569,15 @@ impl PyArray {
     }
 
     #[pymethod]
-    fn frombytes(&self, b: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> {
+    fn frombytes(zelf: PyRef<Self>, b: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
         let b = b.borrow_value();
-        let itemsize = self.borrow_value().itemsize();
+        let itemsize = zelf.borrow_value().itemsize();
         if b.len() % itemsize != 0 {
             return Err(vm.new_value_error("bytes length not a multiple of item size".to_owned()));
         }
         if b.len() / itemsize > 0 {
-            self.borrow_value_mut().frombytes(&b);
+            zelf.borrow_value_mut().frombytes(&b);
         }
         Ok(())
     }
@@ -580,18 +593,20 @@ impl PyArray {
     }
 
     #[pymethod]
-    fn insert(&self, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        let i = saturate_index(i, self.len());
-        self.borrow_value_mut().insert(i, x, vm)
+    fn insert(zelf: PyRef<Self>, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
+        let i = saturate_index(i, zelf.len());
+        zelf.borrow_value_mut().insert(i, x, vm)
     }
 
     #[pymethod]
-    fn pop(&self, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult {
-        if self.len() == 0 {
+    fn pop(zelf: PyRef<Self>, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult {
+        zelf.try_resizable(vm)?;
+        if zelf.len() == 0 {
             Err(vm.new_index_error("pop from empty array".to_owned()))
         } else {
-            let i = self.borrow_value().idx(i.unwrap_or(-1), "pop", vm)?;
-            Ok(self.borrow_value_mut().pop(i, vm))
+            let i = zelf.borrow_value().idx(i.unwrap_or(-1), "pop", vm)?;
+            Ok(zelf.borrow_value_mut().pop(i, vm))
         }
     }
 
@@ -619,9 +634,10 @@ impl PyArray {
     }
 
     #[pymethod]
-    fn fromlist(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+    fn fromlist(zelf: PyRef<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
         if let Some(list) = obj.payload::<PyList>() {
-            self.borrow_value_mut().fromlist(list, vm)
+            zelf.borrow_value_mut().fromlist(list, vm)
         } else {
             Err(vm.new_type_error("arg must be list".to_owned()))
         }
@@ -634,9 +650,7 @@ impl PyArray {
 
     #[pymethod(magic)]
     fn copy(&self) -> PyArray {
-        PyArray {
-            array: PyRwLock::new(self.array.read().clone()),
-        }
+        self.array.read().clone().into()
     }
 
     #[pymethod(magic)]
@@ -656,6 +670,7 @@ impl PyArray {
         obj: PyObjectRef,
         vm: &VirtualMachine,
     ) -> PyResult<()> {
+        // FIXME check resizable
         match needle {
             Either::A(i) => zelf.borrow_value_mut().setitem_by_idx(i, obj, vm),
             Either::B(slice) => {
@@ -684,10 +699,15 @@ impl PyArray {
     }
 
     #[pymethod(name = "__delitem__")]
-    fn delitem(&self, needle: Either<isize, PySliceRef>, vm: &VirtualMachine) -> PyResult<()> {
+    fn delitem(
+        zelf: PyRef<Self>,
+        needle: Either<isize, PySliceRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        zelf.try_resizable(vm)?;
         match needle {
-            Either::A(i) => self.borrow_value_mut().delitem_by_idx(i, vm),
-            Either::B(slice) => self.borrow_value_mut().delitem_by_slice(slice, vm),
+            Either::A(i) => zelf.borrow_value_mut().delitem_by_idx(i, vm),
+            Either::B(slice) => zelf.borrow_value_mut().delitem_by_slice(slice, vm),
         }
     }
 
@@ -696,12 +716,7 @@ impl PyArray {
         if let Some(other) = other.payload::<PyArray>() {
             self.borrow_value()
                 .add(&*other.borrow_value(), vm)
-                .map(|array| {
-                    PyArray {
-                        array: PyRwLock::new(array),
-                    }
-                    .into_ref(vm)
-                })
+                .map(|array| PyArray::from(array).into_ref(vm))
         } else {
             Err(vm.new_type_error(format!(
                 "can only append array (not \"{}\") to array",
@@ -712,6 +727,7 @@ impl PyArray {
 
     #[pymethod(name = "__iadd__")]
     fn iadd(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
+        zelf.try_resizable(vm)?;
         if zelf.is(&other) {
             zelf.borrow_value_mut().imul(2);
             Ok(zelf)
@@ -728,10 +744,7 @@ impl PyArray {
 
     #[pymethod(name = "__mul__")]
     fn mul(&self, counter: isize, vm: &VirtualMachine) -> PyRef<Self> {
-        PyArray {
-            array: PyRwLock::new(self.borrow_value().mul(counter)),
-        }
-        .into_ref(vm)
+        PyArray::from(self.borrow_value().mul(counter)).into_ref(vm)
     }
 
     #[pymethod(name = "__rmul__")]
@@ -740,9 +753,10 @@ impl PyArray {
     }
 
     #[pymethod(name = "__imul__")]
-    fn imul(zelf: PyRef<Self>, counter: isize) -> PyRef<Self> {
+    fn imul(zelf: PyRef<Self>, counter: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
+        zelf.try_resizable(vm)?;
         zelf.borrow_value_mut().imul(counter);
-        zelf
+        Ok(zelf)
     }
 
     #[pymethod(name = "__repr__")]
@@ -836,6 +850,53 @@ impl Comparable for PyArray {
     }
 }
 
+impl BufferProtocol for PyArray {
+    fn get_buffer(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn Buffer>> {
+        zelf.exports.fetch_add(1);
+        Ok(Box::new(zelf))
+    }
+}
+
+impl Buffer for PyArrayRef {
+    fn obj_bytes(&self) -> BorrowedValue<[u8]> {
+        self.get_bytes().into()
+    }
+
+    fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
+        self.get_bytes_mut().into()
+    }
+
+    fn release(&self) {
+        let mut w = self.buffer_options.write();
+        if self.exports.fetch_sub(1) == 1 {
+            *w = None;
+        }
+    }
+
+    fn is_resizable(&self) -> bool {
+        self.exports.load() == 0
+    }
+
+    fn get_options(&self) -> BorrowedValue<BufferOptions> {
+        let guard = self.buffer_options.upgradable_read();
+        let guard = if guard.is_none() {
+            let mut w = PyRwLockUpgradableReadGuard::upgrade(guard);
+            let array = &*self.borrow_value();
+            *w = Some(Box::new(BufferOptions {
+                readonly: false,
+                len: array.len(),
+                itemsize: array.itemsize(),
+                format: array.typecode().to_string(),
+                ..Default::default()
+            }));
+            PyRwLockWriteGuard::downgrade(w)
+        } else {
+            PyRwLockUpgradableReadGuard::downgrade(guard)
+        };
+        PyRwLockReadGuard::map(guard, |x| x.as_ref().unwrap().as_ref()).into()
+    }
+}
+
 #[pyclass(module = "array", name = "array_iterator")]
 #[derive(Debug)]
 pub struct PyArrayIter {

vm/src/stdlib/io.rs

@@ -1402,21 +1402,17 @@ mod _io {
 #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
 mod fileio {
     use super::_io::*;
-    use crate::byteslike::PyBytesLike;
     use crate::exceptions::IntoPyException;
     use crate::function::{OptionalArg, PyFuncArgs};
-    use crate::obj::objbytearray::PyByteArray;
-    use crate::obj::objint;
     use crate::obj::objstr::PyStrRef;
     use crate::obj::objtype::PyTypeRef;
     use crate::pyobject::{
-        BorrowValue, BufferProtocol, Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue,
-        TryFromObject,
+        BorrowValue, Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
     };
     use crate::stdlib::os;
     use crate::vm::VirtualMachine;
+    use crate::{byteslike::PyBytesLike, obj::objmemory::try_buffer_from_object};
     use crossbeam_utils::atomic::AtomicCell;
-    use num_traits::ToPrimitive;
     use std::io::{Read, Seek, SeekFrom, Write};
 
     fn compute_c_flag(mode: &str) -> u32 {
@@ -1571,32 +1567,28 @@ mod fileio {
         }
 
         #[pymethod]
-        fn readinto(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-            if !obj.readonly() {
-                return Err(vm.new_type_error(
+        fn readinto(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> {
+            let buffer = try_buffer_from_object(obj, vm).map_err(|_| {
+                vm.new_type_error(
                     "readinto() argument must be read-write bytes-like object".to_owned(),
-                ));
-            }
+                )
+            })?;
 
-            //extract length of buffer
-            let py_length = vm.call_method(&obj, "__len__", PyFuncArgs::default())?;
-            let length = objint::get_value(&py_length).to_u64().unwrap();
+            let mut writable = buffer
+                .as_contiguous_mut()
+                .ok_or_else(|| vm.new_value_error("buffer is not contiguous".to_owned()))?;
+            let length = writable.len() as u64;
 
             let handle = self.get_file(vm)?;
 
             let mut f = handle.take(length);
-            if let Some(bytes) = obj.payload::<PyByteArray>() {
-                //TODO: Implement for MemoryView
-
-                let value_mut = &mut bytes.borrow_value_mut().elements;
-                value_mut.clear();
-                f.read_to_end(value_mut)
-                    .map_err(|_| vm.new_value_error("Error reading from Take".to_owned()))?;
-            };
+            let ret = f
+                .read(&mut *writable)
+                .map_err(|_| vm.new_value_error("Error reading from Take".to_owned()))?;
 
             self.set_file(f.into_inner())?;
 
-            Ok(())
+            Ok(ret)
         }
 
         #[pymethod]

vm/src/stdlib/marshal.rs

@@ -5,6 +5,7 @@ mod decl {
     use crate::bytecode;
     use crate::obj::objbytes::{PyBytes, PyBytesRef};
     use crate::obj::objcode::{PyCode, PyCodeRef};
+    use crate::obj::objmemory::try_buffer_from_object;
     use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject};
     use crate::vm::VirtualMachine;
 
@@ -20,8 +21,12 @@ mod decl {
     }
 
     #[pyfunction]
-    fn loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<PyCode> {
-        let code = bytecode::CodeObject::from_bytes(&code_bytes)
+    fn loads(code_bytes: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyCode> {
+        let buffer = try_buffer_from_object(code_bytes, vm)?;
+        let bytes = buffer
+            .as_contiguous()
+            .ok_or_else(|| vm.new_value_error("buffer is not contiguous".to_owned()))?;
+        let code = bytecode::CodeObject::from_bytes(&*bytes)
             .map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?;
         Ok(PyCode { code })
     }
@@ -29,7 +34,8 @@ mod decl {
     #[pyfunction]
     fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyCode> {
         let read_res = vm.call_method(&f, "read", vec![])?;
+        // FIXME:
         let bytes = PyBytesRef::try_from_object(vm, read_res)?;
-        loads(bytes, vm)
+        loads(bytes.into_object(), vm)
     }
 }

vm/src/stdlib/mod.rs

@@ -23,7 +23,7 @@ mod marshal;
 mod math;
 mod operator;
 mod platform;
-mod pystruct;
+pub(crate) mod pystruct;
 mod random;
 mod re;
 mod serde_json;

vm/src/stdlib/pystruct.rs

@@ -13,7 +13,7 @@ use crate::pyobject::PyObjectRef;
 use crate::VirtualMachine;
 
 #[pymodule]
-mod _struct {
+pub(crate) mod _struct {
     use byteorder::{ReadBytesExt, WriteBytesExt};
     use crossbeam_utils::atomic::AtomicCell;
     use itertools::Itertools;
@@ -56,11 +56,12 @@ mod _struct {
 
     impl FormatCode {
         fn unit_size(&self) -> usize {
+            // XXX: size of l L q Q is platform depended?
             match self.code {
                 'x' | 'c' | 'b' | 'B' | '?' | 's' | 'p' => 1,
                 'h' | 'H' => 2,
-                'i' | 'l' | 'I' | 'L' | 'f' => 4,
-                'q' | 'Q' | 'd' => 8,
+                'i' | 'I' | 'f' => 4,
+                'l' | 'L' | 'q' | 'Q' | 'd' => 8,
                 'n' | 'N' | 'P' => std::mem::size_of::<usize>(),
                 c => {
                     panic!("Unsupported format code {:?}", c);
@@ -82,7 +83,7 @@ mod _struct {
     }
 
     #[derive(Debug, Clone)]
-    struct FormatSpec {
+    pub(crate) struct FormatSpec {
         endianness: Endianness,
         codes: Vec<FormatCode>,
     }
@@ -104,7 +105,7 @@ mod _struct {
             FormatSpec::parse(decoded_fmt).map_err(|err| new_struct_error(vm, err))
         }
 
-        fn parse(fmt: &str) -> Result<FormatSpec, String> {
+        pub fn parse(fmt: &str) -> Result<FormatSpec, String> {
             let mut chars = fmt.chars().peekable();
 
             // First determine "@", "<", ">","!" or "="
@@ -159,7 +160,7 @@ mod _struct {
             Ok(())
         }
 
-        fn unpack(&self, data: &[u8], vm: &VirtualMachine) -> PyResult<PyTupleRef> {
+        pub fn unpack(&self, data: &[u8], vm: &VirtualMachine) -> PyResult<PyTupleRef> {
             if self.size() != data.len() {
                 return Err(new_struct_error(
                     vm,
@@ -500,8 +501,10 @@ mod _struct {
             '?' => pack_bool,
             'h' => pack_i16::<Endianness>,
             'H' => pack_u16::<Endianness>,
-            'i' | 'l' => pack_i32::<Endianness>,
-            'I' | 'L' => pack_u32::<Endianness>,
+            'i' => pack_i32::<Endianness>,
+            'I' => pack_u32::<Endianness>,
+            'l' => pack_i64::<Endianness>,
+            'L' => pack_u64::<Endianness>,
             'q' => pack_i64::<Endianness>,
             'Q' => pack_u64::<Endianness>,
             'n' => pack_isize::<Endianness>,
@@ -738,8 +741,10 @@ mod _struct {
             '?' => unpack_bool,
             'h' => unpack_i16::<Endianness>,
             'H' => unpack_u16::<Endianness>,
-            'i' | 'l' => unpack_i32::<Endianness>,
-            'I' | 'L' => unpack_u32::<Endianness>,
+            'i' => unpack_i32::<Endianness>,
+            'I' => unpack_u32::<Endianness>,
+            'l' => unpack_i64::<Endianness>,
+            'L' => unpack_u64::<Endianness>,
             'q' => unpack_i64::<Endianness>,
             'Q' => unpack_u64::<Endianness>,
             'n' => unpack_isize::<Endianness>,