Merge pull request #2284 from qingshi163/dev4

generic and efficiency way to consume iterable object

extra_tests/snippets/bytes.py

@@ -620,3 +620,7 @@ class A:
         return b"bytess"
 
 assert bytes(A()) == b"bytess"
+
+# Issue #2125
+b = b'abc'
+assert bytes(b) is b

vm/src/builtins/bytearray.rs

@@ -7,8 +7,9 @@ use super::pytype::PyTypeRef;
 use super::tuple::PyTupleRef;
 use crate::anystr::{self, AnyStr};
 use crate::bytesinner::{
-    bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
-    ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
+    bytes_decode, bytes_from_object, value_from_object, ByteInnerFindOptions, ByteInnerNewOptions,
+    ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs,
+    PyBytesInner,
 };
 use crate::byteslike::PyBytesLike;
 use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
@@ -20,7 +21,7 @@ use crate::pyobject::{
     BorrowValue, Either, IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext,
     PyIterable, PyObjectRef, PyRef, PyResult, PyValue,
 };
-use crate::sliceable::SequenceIndex;
+use crate::sliceable::{PySliceableSequence, PySliceableSequenceMut, SequenceIndex};
 use crate::slots::{BufferProtocol, Comparable, Hashable, PyComparisonOp, Unhashable};
 use crate::vm::VirtualMachine;
 use bstr::ByteSlice;
@@ -107,7 +108,7 @@ impl PyByteArray {
         options: ByteInnerNewOptions,
         vm: &VirtualMachine,
     ) -> PyResult<PyRef<Self>> {
-        PyByteArray::from_inner(options.get_value(vm)?).into_ref_with_type(vm, cls)
+        options.get_bytearray(cls, vm)
     }
 
     #[pymethod(name = "__repr__")]
@@ -139,12 +140,6 @@ impl PyByteArray {
             .new_bytearray(self.borrow_value().add(&*other.borrow_value()))
     }
 
-    #[pymethod(name = "__iadd__")]
-    fn iadd(zelf: PyRef<Self>, other: PyBytesLike, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
-        zelf.try_resizable(vm)?.iadd(&*other.borrow_value());
-        Ok(zelf)
-    }
-
     #[pymethod(name = "__contains__")]
     fn contains(
         &self,
@@ -154,12 +149,7 @@ impl PyByteArray {
         self.borrow_value().contains(needle, vm)
     }
 
-    #[pymethod(name = "__getitem__")]
-    fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult {
-        self.borrow_value().getitem("bytearray", needle, vm)
-    }
-
-    #[pymethod(name = "__setitem__")]
+    #[pymethod(magic)]
     fn setitem(
         zelf: PyRef<Self>,
         needle: SequenceIndex,
@@ -167,27 +157,147 @@ impl PyByteArray {
         vm: &VirtualMachine,
     ) -> PyResult<()> {
         match needle {
-            SequenceIndex::Int(int) => zelf.borrow_value_mut().setindex(int, value, vm),
-            SequenceIndex::Slice(slice) => {
-                if let Ok(mut w) = zelf.try_resizable(vm) {
-                    if zelf.is(&value) {
-                        w.setslice_from_self(slice, vm)
-                    } else {
-                        w.setslice(slice, value, vm)
-                    }
-                } else if zelf.is(&value) {
-                    zelf.borrow_value_mut()
-                        .setslice_from_self_no_resize(slice, vm)
+            SequenceIndex::Int(i) => {
+                let value = value_from_object(vm, &value)?;
+                let elements = &mut zelf.borrow_value_mut().elements;
+                if let Some(i) = elements.wrap_index(i) {
+                    elements[i] = value;
+                    Ok(())
                 } else {
-                    zelf.borrow_value_mut().setslice_no_resize(slice, value, vm)
+                    Err(vm.new_index_error("index out of range".to_owned()))
+                }
+            }
+            SequenceIndex::Slice(slice) => {
+                let items = if zelf.is(&value) {
+                    zelf.borrow_value().elements.clone()
+                } else {
+                    bytes_from_object(vm, &value)?
+                };
+                if let Ok(mut w) = zelf.try_resizable(vm) {
+                    w.elements.set_slice_items(vm, &slice, items.as_slice())
+                } else {
+                    zelf.borrow_value_mut().elements.set_slice_items_no_resize(
+                        vm,
+                        &slice,
+                        items.as_slice(),
+                    )
                 }
             }
         }
     }
 
-    #[pymethod(name = "__delitem__")]
-    fn delitem(zelf: PyRef<Self>, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> {
-        zelf.try_resizable(vm)?.delitem(needle, vm)
+    #[pymethod(magic)]
+    fn iadd(zelf: PyRef<Self>, other: PyBytesLike, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
+        zelf.try_resizable(vm)?
+            .elements
+            .extend(&*other.borrow_value());
+        Ok(zelf)
+    }
+
+    #[pymethod(magic)]
+    fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult {
+        self.borrow_value().getitem("bytearray", needle, vm)
+    }
+
+    #[pymethod(magic)]
+    pub fn delitem(&self, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> {
+        let elements = &mut self.try_resizable(vm)?.elements;
+        match needle {
+            SequenceIndex::Int(int) => {
+                if let Some(idx) = elements.wrap_index(int) {
+                    elements.remove(idx);
+                    Ok(())
+                } else {
+                    Err(vm.new_index_error("index out of range".to_owned()))
+                }
+            }
+            SequenceIndex::Slice(slice) => elements.delete_slice(vm, &slice),
+        }
+    }
+
+    #[pymethod]
+    fn pop(zelf: PyRef<Self>, index: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<u8> {
+        let elements = &mut zelf.try_resizable(vm)?.elements;
+        let index = elements
+            .wrap_index(index.unwrap_or(-1))
+            .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?;
+        Ok(elements.remove(index))
+    }
+
+    #[pymethod]
+    fn insert(
+        zelf: PyRef<Self>,
+        index: isize,
+        object: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        let value = value_from_object(vm, &object)?;
+        let elements = &mut zelf.try_resizable(vm)?.elements;
+        let index = elements.saturate_index(index);
+        elements.insert(index, value);
+        Ok(())
+    }
+
+    #[pymethod]
+    fn append(zelf: PyRef<Self>, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        let value = value_from_object(vm, &object)?;
+        zelf.try_resizable(vm)?.elements.push(value);
+        Ok(())
+    }
+
+    #[pymethod]
+    fn remove(zelf: PyRef<Self>, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        let value = value_from_object(vm, &object)?;
+        let elements = &mut zelf.try_resizable(vm)?.elements;
+        if let Some(index) = elements.find_byte(value) {
+            elements.remove(index);
+            Ok(())
+        } else {
+            Err(vm.new_value_error("value not found in bytearray".to_owned()))
+        }
+    }
+
+    #[pymethod]
+    fn extend(zelf: PyRef<Self>, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        if zelf.is(&object) {
+            Self::irepeat(&zelf, 2, vm)
+        } else {
+            let items = bytes_from_object(vm, &object)?;
+            zelf.try_resizable(vm)?.elements.extend(items);
+            Ok(())
+        }
+    }
+
+    fn irepeat(zelf: &PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<()> {
+        if n == 1 {
+            return Ok(());
+        }
+        let mut w = match zelf.try_resizable(vm) {
+            Ok(w) => w,
+            Err(err) => {
+                return if zelf.borrow_value().elements.is_empty() {
+                    // We can multiple an empty vector by any integer
+                    Ok(())
+                } else {
+                    Err(err)
+                };
+            }
+        };
+        let elements = &mut w.elements;
+
+        if n <= 0 {
+            elements.clear();
+        } else if n != 1 {
+            let n = n as usize;
+
+            let old = elements.clone();
+
+            elements.reserve((n - 1) * old.len());
+            for _ in 1..n {
+                elements.extend(&old);
+            }
+        }
+        Ok(())
     }
 
     #[pymethod(name = "isalnum")]
@@ -352,11 +462,6 @@ impl PyByteArray {
         index.ok_or_else(|| vm.new_value_error("substring not found".to_owned()))
     }
 
-    #[pymethod(name = "remove")]
-    fn remove(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.try_resizable(vm)?.remove(value, vm)
-    }
-
     #[pymethod(name = "translate")]
     fn translate(
         &self,
@@ -483,37 +588,6 @@ impl PyByteArray {
         self.borrow_value().elements.clone().into()
     }
 
-    #[pymethod(name = "append")]
-    fn append(zelf: PyRef<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        zelf.try_resizable(vm)?.append(value, vm)
-    }
-
-    #[pymethod(name = "extend")]
-    fn extend(zelf: PyRef<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        if zelf.is(&value) {
-            zelf.try_resizable(vm)?.irepeat(2);
-            Ok(())
-        } else {
-            zelf.try_resizable(vm)?.extend(value, vm)
-        }
-    }
-
-    #[pymethod(name = "insert")]
-    fn insert(
-        zelf: PyRef<Self>,
-        index: isize,
-        value: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        zelf.try_resizable(vm)?.insert(index, value, vm)
-    }
-
-    #[pymethod(name = "pop")]
-    fn pop(zelf: PyRef<Self>, index: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<u8> {
-        let index = index.unwrap_or(-1);
-        zelf.try_resizable(vm)?.pop(index, vm)
-    }
-
     #[pymethod(name = "title")]
     fn title(&self) -> Self {
         self.borrow_value().title().into()
@@ -525,10 +599,9 @@ impl PyByteArray {
         self.borrow_value().repeat(n).into()
     }
 
-    #[pymethod(name = "__imul__")]
+    #[pymethod(magic)]
     fn imul(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
-        zelf.try_resizable(vm)?.irepeat(n);
-        Ok(zelf)
+        Self::irepeat(&zelf, n, vm).map(|_| zelf)
     }
 
     #[pymethod(name = "__mod__")]

vm/src/builtins/bytes.rs

@@ -111,11 +111,7 @@ impl PyBytes {
         options: ByteInnerNewOptions,
         vm: &VirtualMachine,
     ) -> PyResult<PyRef<Self>> {
-        PyBytes {
-            inner: options.get_value(vm)?,
-            buffer_options: OnceCell::new(),
-        }
-        .into_ref_with_type(vm, cls)
+        options.get_bytes(cls, vm)
     }
 
     #[pymethod(name = "__repr__")]

vm/src/builtins/list.rs

@@ -4,12 +4,9 @@ use std::mem::size_of;
 use std::ops::DerefMut;
 
 use crossbeam_utils::atomic::AtomicCell;
-use num_traits::ToPrimitive;
 
-use super::int::PyIntRef;
 use super::pytype::PyTypeRef;
 use super::slice::PySliceRef;
-use crate::bytesinner;
 use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard};
 use crate::function::OptionalArg;
 use crate::pyobject::{
@@ -70,25 +67,6 @@ impl PyList {
     pub fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, Vec<PyObjectRef>> {
         self.elements.write()
     }
-
-    // TODO: more generic way to do so
-    pub(crate) fn to_byte_inner(&self, vm: &VirtualMachine) -> PyResult<bytesinner::PyBytesInner> {
-        let mut elements = Vec::<u8>::with_capacity(self.borrow_value().len());
-        for elem in self.borrow_value().iter() {
-            let py_int = PyIntRef::try_from_object(vm, elem.clone()).map_err(|_| {
-                vm.new_type_error(format!(
-                    "'{}' object cannot be interpreted as an integer",
-                    elem.class().name
-                ))
-            })?;
-            let result = py_int
-                .borrow_value()
-                .to_u8()
-                .ok_or_else(|| vm.new_value_error("bytes must be in range (0, 256)".to_owned()))?;
-            elements.push(result);
-        }
-        Ok(elements.into())
-    }
 }
 
 #[derive(FromArgs, Default)]

vm/src/builtins/tuple.rs

@@ -1,20 +1,17 @@
 use crossbeam_utils::atomic::AtomicCell;
-use num_traits::ToPrimitive;
 use std::fmt;
 
-use super::int::PyIntRef;
 use super::pytype::PyTypeRef;
 use crate::common::hash::PyHash;
+use crate::function::OptionalArg;
 use crate::pyobject::{
     self, BorrowValue, Either, IdProtocol, IntoPyObject, PyArithmaticValue, PyClassImpl,
-    PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
-    TypeProtocol,
+    PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol,
 };
 use crate::sequence::{self, SimpleSeq};
 use crate::sliceable::PySliceableSequence;
 use crate::slots::{Comparable, Hashable, PyComparisonOp};
 use crate::vm::{ReprGuard, VirtualMachine};
-use crate::{bytesinner::PyBytesInner, function::OptionalArg};
 
 /// tuple() -> empty tuple
 /// tuple(iterable) -> tuple initialized from iterable's items
@@ -68,25 +65,6 @@ impl PyTuple {
     pub(crate) fn fast_getitem(&self, idx: usize) -> PyObjectRef {
         self.elements[idx].clone()
     }
-
-    // TODO: more generic way to do so
-    pub(crate) fn to_bytes_inner(&self, vm: &VirtualMachine) -> PyResult<PyBytesInner> {
-        let mut elements = Vec::<u8>::with_capacity(self.borrow_value().len());
-        for elem in self.borrow_value().iter() {
-            let py_int = PyIntRef::try_from_object(vm, elem.clone()).map_err(|_| {
-                vm.new_type_error(format!(
-                    "'{}' object cannot be interpreted as an integer",
-                    elem.class().name
-                ))
-            })?;
-            let result = py_int
-                .borrow_value()
-                .to_u8()
-                .ok_or_else(|| vm.new_value_error("bytes must be in range (0, 256)".to_owned()))?;
-            elements.push(result);
-        }
-        Ok(elements.into())
-    }
 }
 
 pub type PyTupleRef = PyRef<PyTuple>;

vm/src/bytesinner.rs

@@ -4,22 +4,19 @@ use num_bigint::BigInt;
 use num_traits::ToPrimitive;
 
 use crate::anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper};
-use crate::builtins::bytearray::PyByteArray;
+use crate::builtins::bytearray::{PyByteArray, PyByteArrayRef};
 use crate::builtins::bytes::{PyBytes, PyBytesRef};
-use crate::builtins::int::{self, PyInt, PyIntRef};
-use crate::builtins::list::PyList;
-use crate::builtins::memory::PyMemoryView;
+use crate::builtins::int::{PyInt, PyIntRef};
 use crate::builtins::pystr::{self, PyStr, PyStrRef};
 use crate::builtins::singletons::PyNoneRef;
-use crate::builtins::slice::PySliceRef;
-use crate::builtins::tuple::PyTuple;
+use crate::builtins::PyTypeRef;
 use crate::byteslike::try_bytes_like;
 use crate::function::{OptionalArg, OptionalOption};
 use crate::pyobject::{
-    BorrowValue, Either, PyComparisonValue, PyIterable, PyIterator, PyObjectRef, PyResult,
-    TryFromObject, TypeProtocol,
+    BorrowValue, Either, IdProtocol, PyComparisonValue, PyIterable, PyObjectRef, PyRef, PyResult,
+    PyValue, TryFromObject, TypeProtocol,
 };
-use crate::sliceable::{PySliceableSequence, PySliceableSequenceMut, SequenceIndex};
+use crate::sliceable::PySliceableSequence;
 use crate::slots::PyComparisonOp;
 use crate::vm::VirtualMachine;
 use rustpython_common::hash;
@@ -37,27 +34,7 @@ impl From<Vec<u8>> for PyBytesInner {
 
 impl TryFromObject for PyBytesInner {
     fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
-        if let Ok(zelf) = try_bytes_like(vm, &obj, |bytes| Self::from(bytes.to_vec())) {
-            return Ok(zelf);
-        }
-
-        match_class!(match obj {
-            // TODO: generic way from &[PyObjectRef]
-            l @ PyList => l.to_byte_inner(vm),
-            t @ PyTuple => t.to_bytes_inner(vm),
-            obj => {
-                let iter = vm.get_method_or_type_error(obj.clone(), "__iter__", || {
-                    format!(
-                        "a bytes-like object is required, not '{}'",
-                        obj.class().name
-                    )
-                })?;
-                let iter = PyIterable::from_method(iter);
-                Ok(PyBytesInner {
-                    elements: iter.iter(vm)?.collect::<PyResult<_>>()?,
-                })
-            }
-        })
+        bytes_from_object(vm, &obj).map(Self::from)
     }
 }
 
@@ -72,83 +49,98 @@ pub struct ByteInnerNewOptions {
 }
 
 impl ByteInnerNewOptions {
-    pub fn get_value(self, vm: &VirtualMachine) -> PyResult<PyBytesInner> {
-        match self.source {
-            OptionalArg::Missing => {
-                if let OptionalArg::Present(_) = self.encoding {
-                    Err(vm.new_type_error("encoding without a string argument".to_owned()))
-                } else if let OptionalArg::Present(_) = self.errors {
-                    Err(vm.new_type_error("errors without a string argument".to_owned()))
-                } else {
-                    Ok(PyBytesInner {
-                        elements: Vec::new(),
-                    })
-                }
-            }
-            OptionalArg::Present(obj) => {
-                match obj.downcast::<PyStr>() {
-                    Ok(s) => {
-                        // Handle bytes(string, encoding[, errors])
-                        if let OptionalArg::Present(enc) = self.encoding {
-                            let bytes =
-                                pystr::encode_string(s, Some(enc), self.errors.into_option(), vm)?;
-                            Ok(PyBytesInner {
-                                elements: bytes.borrow_value().to_vec(),
-                            })
-                        } else {
-                            Err(vm.new_type_error("string argument without an encoding".to_owned()))
-                        }
-                    }
-                    Err(obj) => {
-                        if let OptionalArg::Present(_) = self.encoding {
-                            Err(vm.new_type_error("encoding without a string argument".to_owned()))
-                        } else if let OptionalArg::Present(_) = self.errors {
-                            Err(vm.new_type_error("errors without a string argument".to_owned()))
-                        } else {
-                            let value = match_class!(match obj {
-                                i @ PyInt => {
-                                    let size = int::get_value(&i.into_object())
-                                        .to_isize()
-                                        .ok_or_else(|| {
-                                            vm.new_overflow_error(
-                                                "cannot fit 'int' into an index-sized integer"
-                                                    .to_owned(),
-                                            )
-                                        })?;
-                                    let size = if size < 0 {
-                                        return Err(vm.new_value_error("negative count".to_owned()));
-                                    } else {
-                                        size as usize
-                                    };
-                                    Ok(vec![0; size])
-                                }
-                                i @ PyBytes => Ok(i.borrow_value().to_vec()),
-                                j @ PyByteArray => Ok(j.borrow_value().elements.to_vec()),
-                                obj => {
-                                    // TODO: only support this method in the bytes() constructor
-                                    if let Some(bytes_method) =
-                                        vm.get_method(obj.clone(), "__bytes__")
-                                    {
-                                        let bytes = vm.invoke(&bytes_method?, ())?;
-                                        return PyBytesInner::try_from_object(vm, bytes);
-                                    }
-                                    PyBytesInner::value_seq_try_from_object(vm, obj.clone())
-                                    // TODO: better error message
-                                    // .map_err(|_| {
-                                    //     vm.new_type_error(format!(
-                                    //         "cannot convert '{}' object to bytes",
-                                    //         obj.class().name
-                                    //     ))
-                                    // })?;
-                                }
-                            });
+    fn get_value_from_string(
+        s: PyStrRef,
+        encoding: OptionalArg<PyStrRef>,
+        errors: OptionalArg<PyStrRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyBytesInner> {
+        // Handle bytes(string, encoding[, errors])
+        let encoding = encoding
+            .ok_or_else(|| vm.new_type_error("string argument without an encoding".to_owned()))?;
+        let bytes = pystr::encode_string(s, Some(encoding), errors.into_option(), vm)?;
+        Ok(bytes.borrow_value().to_vec().into())
+    }
 
-                            value.map(|v| PyBytesInner { elements: v })
-                        }
+    fn get_value_from_source(source: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> {
+        bytes_from_object(vm, &source).map(|x| x.into())
+    }
+
+    fn get_value_from_size(size: PyIntRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> {
+        let size = size.borrow_value().to_isize().ok_or_else(|| {
+            vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_owned())
+        })?;
+        let size = if size < 0 {
+            return Err(vm.new_value_error("negative count".to_owned()));
+        } else {
+            size as usize
+        };
+        Ok(vec![0; size].into())
+    }
+
+    fn check_args(self, vm: &VirtualMachine) -> PyResult<()> {
+        if let OptionalArg::Present(_) = self.encoding {
+            Err(vm.new_type_error("encoding without a string argument".to_owned()))
+        } else if let OptionalArg::Present(_) = self.errors {
+            Err(vm.new_type_error("errors without a string argument".to_owned()))
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn get_bytes(mut self, cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<PyBytesRef> {
+        let inner = if let OptionalArg::Present(source) = self.source.take() {
+            if source.class().is(&PyBytes::class(vm)) && cls.is(&PyBytes::class(vm)) {
+                return self
+                    .check_args(vm)
+                    .map(|_| unsafe { PyRef::from_obj_unchecked(source) });
+            }
+
+            match_class!(match source {
+                s @ PyStr => Self::get_value_from_string(s, self.encoding, self.errors, vm),
+                i @ PyInt => {
+                    self.check_args(vm)?;
+                    Self::get_value_from_size(i, vm)
+                }
+                obj => {
+                    self.check_args(vm)?;
+                    if let Some(bytes_method) = vm.get_method(obj.clone(), "__bytes__") {
+                        let bytes = vm.invoke(&bytes_method?, ())?;
+                        PyBytesInner::try_from_object(vm, bytes)
+                    } else {
+                        Self::get_value_from_source(obj, vm)
                     }
                 }
-            }
-        }
+            })
+        } else {
+            self.check_args(vm).map(|_| vec![].into())
+        }?;
+
+        PyBytes::from(inner).into_ref_with_type(vm, cls)
+    }
+
+    pub fn get_bytearray(
+        mut self,
+        cls: PyTypeRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyByteArrayRef> {
+        let inner = if let OptionalArg::Present(source) = self.source.take() {
+            match_class!(match source {
+                s @ PyStr => Self::get_value_from_string(s, self.encoding, self.errors, vm),
+                i @ PyInt => {
+                    self.check_args(vm)?;
+                    Self::get_value_from_size(i, vm)
+                }
+                obj => {
+                    self.check_args(vm)?;
+                    Self::get_value_from_source(obj, vm)
+                }
+            })
+        } else {
+            self.check_args(vm).map(|_| vec![].into())
+        }?;
+
+        PyByteArray::from(inner).into_ref_with_type(vm, cls)
     }
 }
 
@@ -280,10 +272,6 @@ impl PyBytesInner {
         self.elements.py_add(other)
     }
 
-    pub fn iadd(&mut self, other: &[u8]) {
-        self.elements.extend(other);
-    }
-
     pub fn contains(
         &self,
         needle: Either<PyBytesInner, PyIntRef>,
@@ -308,146 +296,6 @@ impl PyBytesInner {
         Ok(obj)
     }
 
-    pub fn setindex(
-        &mut self,
-        int: isize,
-        object: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        if let Some(idx) = self.elements.wrap_index(int) {
-            let value = Self::value_try_from_object(vm, object)?;
-            self.elements[idx] = value;
-            Ok(())
-        } else {
-            Err(vm.new_index_error("index out of range".to_owned()))
-        }
-    }
-
-    fn value_try_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<u8> {
-        let value = vm.to_index(&object).ok_or_else(|| {
-            vm.new_type_error(format!(
-                "'{}' object cannot be interpreted as an integer",
-                object.class().name
-            ))
-        })?;
-        // __index__ returned non-int type
-        let value = value?;
-        value
-            .borrow_value()
-            .to_u8()
-            .ok_or_else(|| vm.new_value_error("byte must be in range(0, 256)".to_owned()))
-    }
-
-    fn value_seq_try_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Vec<u8>> {
-        if let Ok(iterable) = PyIterable::try_from_object(vm, object.clone()) {
-            let iter: PyIterator<PyObjectRef> = iterable.iter(vm)?;
-            iter.map(|obj| Self::value_try_from_object(vm, obj?))
-                .try_collect()
-        } else if let Some(mview) = object.payload_if_subclass::<PyMemoryView>(vm) {
-            mview.try_bytes(vm, |v| v.to_vec())
-        } else {
-            Err(vm.new_type_error(
-                "can assign only bytes, buffers, or iterables of ints in range(0, 256)".to_owned(),
-            ))
-        }
-    }
-
-    pub fn setslice(
-        &mut self,
-        slice: PySliceRef,
-        object: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        let items = Self::value_seq_try_from_object(vm, object)?;
-        self.elements.set_slice_items(vm, &slice, items.as_slice())
-    }
-
-    pub fn setslice_no_resize(
-        &mut self,
-        slice: PySliceRef,
-        object: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        let items = Self::value_seq_try_from_object(vm, object)?;
-        self.elements
-            .set_slice_items_no_resize(vm, &slice, items.as_slice())
-    }
-
-    pub fn setslice_from_self(&mut self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult<()> {
-        let items = self.elements.clone();
-        self.elements.set_slice_items(vm, &slice, items.as_slice())
-    }
-
-    pub fn setslice_from_self_no_resize(
-        &mut self,
-        slice: PySliceRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        let items = self.elements.clone();
-        self.elements
-            .set_slice_items_no_resize(vm, &slice, items.as_slice())
-    }
-
-    pub fn delitem(&mut self, needle: SequenceIndex, vm: &VirtualMachine) -> PyResult<()> {
-        match needle {
-            SequenceIndex::Int(int) => {
-                if let Some(idx) = self.elements.wrap_index(int) {
-                    self.elements.remove(idx);
-                    Ok(())
-                } else {
-                    Err(vm.new_index_error("index out of range".to_owned()))
-                }
-            }
-            SequenceIndex::Slice(slice) => self.delslice(slice, vm),
-        }
-    }
-
-    fn delslice(&mut self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult<()> {
-        self.elements.delete_slice(vm, &slice)
-    }
-
-    pub fn pop(&mut self, index: isize, vm: &VirtualMachine) -> PyResult<u8> {
-        if let Some(index) = self.elements.wrap_index(index) {
-            Ok(self.elements.remove(index))
-        } else {
-            Err(vm.new_index_error("index out of range".to_owned()))
-        }
-    }
-
-    pub fn insert(
-        &mut self,
-        index: isize,
-        object: PyObjectRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<()> {
-        let value = Self::value_try_from_object(vm, object)?;
-        let index = self.elements.saturate_index(index);
-        self.elements.insert(index, value);
-        Ok(())
-    }
-
-    pub fn append(&mut self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        let value = Self::value_try_from_object(vm, object)?;
-        self.elements.push(value);
-        Ok(())
-    }
-
-    pub fn remove(&mut self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        let value = Self::value_try_from_object(vm, object)?;
-        if let Some(index) = self.elements.find_byte(value) {
-            self.elements.remove(index);
-            Ok(())
-        } else {
-            Err(vm.new_value_error("value not found in bytearray".to_owned()))
-        }
-    }
-
-    pub fn extend(&mut self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
-        let items = Self::value_seq_try_from_object(vm, object)?;
-        self.elements.extend(items);
-        Ok(())
-    }
-
     pub fn isalnum(&self) -> bool {
         !self.elements.is_empty()
             && self
@@ -1053,26 +901,6 @@ impl PyBytesInner {
     pub fn repeat(&self, n: isize) -> Vec<u8> {
         self.elements.repeat(n.to_usize().unwrap_or(0))
     }
-
-    pub fn irepeat(&mut self, n: isize) {
-        if self.elements.is_empty() {
-            // We can multiple an empty vector by any integer, even if it doesn't fit in an isize.
-            return;
-        }
-
-        if n <= 0 {
-            self.elements.clear();
-        } else if n != 1 {
-            let n = n as usize;
-
-            let old = self.elements.clone();
-
-            self.elements.reserve((n - 1) * old.len());
-            for _ in 1..n {
-                self.elements.extend(&old);
-            }
-        }
-    }
 }
 
 pub fn try_as_bytes<F, R>(obj: PyObjectRef, f: F) -> Option<R>
@@ -1345,3 +1173,32 @@ pub fn bytes_to_hex(
 const fn is_py_ascii_whitespace(b: u8) -> bool {
     matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
 }
+
+pub fn bytes_from_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Vec<u8>> {
+    if let Ok(elements) = try_bytes_like(vm, obj, |bytes| bytes.to_vec()) {
+        return Ok(elements);
+    }
+
+    if let Ok(elements) = vm.map_iterable_object(obj, |x| value_from_object(vm, &x)) {
+        return Ok(elements?);
+    }
+
+    Err(vm.new_type_error(
+        "can assign only bytes, buffers, or iterables of ints in range(0, 256)".to_owned(),
+    ))
+}
+
+pub fn value_from_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<u8> {
+    let value = vm.to_index(obj).ok_or_else(|| {
+        vm.new_type_error(format!(
+            "'{}' object cannot be interpreted as an integer",
+            obj.class().name
+        ))
+    })?;
+    // __index__ returned non-int type
+    let value = value?;
+    value
+        .borrow_value()
+        .to_u8()
+        .ok_or_else(|| vm.new_value_error("byte must be in range(0, 256)".to_owned()))
+}

vm/src/iterator.rs

@@ -78,6 +78,24 @@ pub fn get_all<T: TryFromObject>(vm: &VirtualMachine, iter_obj: &PyObjectRef) ->
     Ok(elements)
 }
 
+pub fn try_map<F, R>(vm: &VirtualMachine, iter_obj: &PyObjectRef, mut f: F) -> PyResult<Vec<R>>
+where
+    F: FnMut(PyObjectRef) -> PyResult<R>,
+{
+    let cap = length_hint(vm, iter_obj.clone())?.unwrap_or(0);
+    // TODO: fix extend to do this check (?), see test_extend in Lib/test/list_tests.py,
+    // https://github.com/python/cpython/blob/v3.9.0/Objects/listobject.c#L922-L928
+    if cap >= isize::max_value() as usize {
+        return Ok(Vec::new());
+    }
+    let mut results = Vec::with_capacity(cap);
+    while let Some(element) = get_next_object(vm, iter_obj)? {
+        results.push(f(element)?);
+    }
+    results.shrink_to_fit();
+    Ok(results)
+}
+
 pub fn stop_iter_with_value(val: PyObjectRef, vm: &VirtualMachine) -> PyBaseExceptionRef {
     let stop_iteration_type = vm.ctx.exceptions.stop_iteration.clone();
     vm.new_exception(stop_iteration_type, vec![val])

vm/src/vm.rs

@@ -992,6 +992,45 @@ impl VirtualMachine {
         }
     }
 
+    pub fn map_iterable_object<F, R>(
+        &self,
+        obj: &PyObjectRef,
+        mut f: F,
+    ) -> PyResult<PyResult<Vec<R>>>
+    where
+        F: FnMut(PyObjectRef) -> PyResult<R>,
+    {
+        match_class!(match obj {
+            ref l @ PyList => {
+                let mut i: usize = 0;
+                let mut results = Vec::with_capacity(l.borrow_value().len());
+                loop {
+                    let elem = {
+                        let elements = &*l.borrow_value();
+                        if i >= elements.len() {
+                            results.shrink_to_fit();
+                            return Ok(Ok(results));
+                        } else {
+                            elements[i].clone()
+                        }
+                        // free the lock
+                    };
+                    match f(elem) {
+                        Ok(result) => results.push(result),
+                        Err(err) => return Ok(Err(err)),
+                    }
+                    i += 1;
+                }
+            }
+            ref t @ PyTuple => Ok(t.borrow_value().iter().cloned().map(f).collect()),
+            // TODO: put internal iterable type
+            obj => {
+                let iter = iterator::get_iter(self, obj)?;
+                Ok(iterator::try_map(self, &iter, f))
+            }
+        })
+    }
+
     // get_attribute should be used for full attribute access (usually from user code).
     #[cfg_attr(feature = "flame-it", flame("VirtualMachine"))]
     pub fn get_attribute<T>(&self, obj: PyObjectRef, attr_name: T) -> PyResult