mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Reimpl Buffer Protocol and memoryview support ndarray with shape, stride and suboffset (#3340)

Browse Source 
* Fix buffer protocol and memoryview

* Fix io and array the right way to use buffer protocol

* fix rebase

* fix memoryview and introduce VecBuffer

* fix deadlock

* fix memoryview avoid double release buffer

* impl ndim buffer support

* first implement nd array for buffer and memoryview

* fix slice adjust_indices

* fix adjust_indices introduct SaturatedSliceIterator

* fix memoryview cmp

* fix adjust_indices positive_order

* fix bug mark passed test

* fix clippy

* fix set_item_slice deadlock, optimize buffer

* Delete @test_113974_tmp

* Remove vec_buffer_type

* rusty names

* impl memoryview multi index

* add comments

Co-authored-by: Jeong YunWon <jeong@youknowone.org>
This commit is contained in:
Steve ShiandJeong YunWon authored and GitHub committed 2021-11-13 07:17:11 +02:00
1 parent 5ceb2be86e
commit 742ea0c4fa
17 files changed
+1417 -1071
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Lib/test/test_gzip.py
Vendored
-2
View File
@@ -88,8 +88,6 @@ class TestGzip(BaseTest):
self.assertEqual(d, data1 * 51)
self.assertIsInstance(f.name, str)
# TODO: RUSTPYTHON
@unittest.expectedFailure
# The following test_write_xy methods test that write accepts
# the corresponding bytes-like object type as input
# and that the data written equals bytes(xy) in all cases.
Lib/test/test_memoryview.py
Vendored
-2
View File
@@ -71,8 +71,6 @@ class AbstractMemoryTests:
m = None
self.assertEqual(sys.getrefcount(b), oldrefcount)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_setitem_writable(self):
if not self.rw_type:
self.skipTest("no writable type to test")
extra_tests/snippets/memoryview.py
+4 -2
View File
@@ -45,6 +45,8 @@ def test_slice():
assert m[::2][1:-1].tobytes() == b'357'
assert m[::2][::2].tobytes() == b'159'
assert m[::2][1::2].tobytes() == b'37'
assert m[::-1].tobytes() == b'987654321'
assert m[::-2].tobytes() == b'97531'
test_slice()
@@ -56,9 +58,9 @@ def test_resizable():
m.release()
b.append(6)
m2 = memoryview(b)
m4 = memoryview(b)
m4 = memoryview(m2)
assert_raises(BufferError, lambda: b.append(5))
m3 = memoryview(b)
m3 = memoryview(m2)
assert_raises(BufferError, lambda: b.append(5))
m2.release()
assert_raises(BufferError, lambda: b.append(5))
stdlib/src/array.rs
+34 -40
View File
@@ -3,7 +3,7 @@ pub(crate) use array::make_module;
#[pymodule(name = "array")]
mod array {
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
atomic::{self, AtomicUsize},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyRwLock, PyRwLockReadGuard,
PyRwLockWriteGuard,
@@ -20,7 +20,7 @@ mod array {
ArgBytesLike, ArgIntoFloat, ArgIterable, IntoPyObject, IntoPyResult, OptionalArg,
},
protocol::{
BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer, PyIterReturn,
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods,
},
sliceable::{PySliceableSequence, PySliceableSequenceMut, SaturatedSlice, SequenceIndex},
@@ -28,10 +28,9 @@ mod array {
AsBuffer, AsMapping, Comparable, Constructor, IterNext, IterNextIterable, Iterable,
PyComparisonOp,
},
IdProtocol, PyComparisonValue, PyObject, PyObjectRef, PyObjectView, PyRef, PyResult,
PyValue, TryFromObject, TypeProtocol, VirtualMachine,
IdProtocol, PyComparisonValue, PyObject, PyObjectRef, PyObjectView, PyObjectWrap, PyRef,
PyResult, PyValue, TryFromObject, TypeProtocol, VirtualMachine,
};
use crossbeam_utils::atomic::AtomicCell;
use itertools::Itertools;
use num_traits::ToPrimitive;
use std::cmp::Ordering;
@@ -615,7 +614,7 @@ mod array {
#[derive(Debug, PyValue)]
pub struct PyArray {
array: PyRwLock<ArrayContentType>,
exports: AtomicCell<usize>,
exports: AtomicUsize,
}
pub type PyArrayRef = PyRef<PyArray>;
@@ -624,7 +623,7 @@ mod array {
fn from(array: ArrayContentType) -> Self {
PyArray {
array: PyRwLock::new(array),
exports: AtomicCell::new(0),
exports: AtomicUsize::new(0),
}
}
}
@@ -1220,40 +1219,35 @@ mod array {
fn as_buffer(zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let array = zelf.read();
let buf = PyBuffer::new(
zelf.as_object().to_owned(),
PyArrayBufferInternal(zelf.to_owned()),
BufferOptions {
readonly: false,
len: array.len(),
itemsize: array.itemsize(),
format: array.typecode_str().into(),
..Default::default()
},
zelf.to_owned().into(),
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
),
&BUFFER_METHODS,
);
Ok(buf)
}
}
#[derive(Debug)]
struct PyArrayBufferInternal(PyRef<PyArray>);
impl BufferInternal for PyArrayBufferInternal {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.0.get_bytes().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.0.get_bytes_mut().into()
}
fn release(&self) {
self.0.exports.fetch_sub(1);
}
fn retain(&self) {
self.0.exports.fetch_add(1);
}
}
static BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| buffer.obj_as::<PyArray>().get_bytes().into(),
obj_bytes_mut: |buffer| buffer.obj_as::<PyArray>().get_bytes_mut().into(),
release: |buffer| {
buffer
.obj_as::<PyArray>()
.exports
.fetch_sub(1, atomic::Ordering::Release);
},
retain: |buffer| {
buffer
.obj_as::<PyArray>()
.exports
.fetch_add(1, atomic::Ordering::Release);
},
};
impl AsMapping for PyArray {
fn as_mapping(_zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyMappingMethods {
@@ -1290,7 +1284,7 @@ mod array {
impl Iterable for PyArray {
fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult {
Ok(PyArrayIter {
position: AtomicCell::new(0),
position: AtomicUsize::new(0),
array: zelf,
}
.into_object(vm))
@@ -1302,7 +1296,7 @@ mod array {
fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult<Self::Resizable> {
let w = self.write();
if self.exports.load() == 0 {
if self.exports.load(atomic::Ordering::SeqCst) == 0 {
Ok(w)
} else {
Err(vm.new_buffer_error(
@@ -1316,7 +1310,7 @@ mod array {
#[pyclass(name = "array_iterator")]
#[derive(Debug, PyValue)]
pub struct PyArrayIter {
position: AtomicCell<usize>,
position: AtomicUsize,
array: PyArrayRef,
}
@@ -1326,7 +1320,7 @@ mod array {
impl IterNextIterable for PyArrayIter {}
impl IterNext for PyArrayIter {
fn next(zelf: &PyObjectView<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
let pos = zelf.position.fetch_add(1);
let pos = zelf.position.fetch_add(1, atomic::Ordering::SeqCst);
let r = if let Some(item) = zelf.array.read().getitem_by_idx(pos, vm)? {
PyIterReturn::Return(item)
} else {
vm/src/builtins/bytearray.rs
+47 -42
View File
@@ -3,13 +3,6 @@ use super::{
PositionIterInternal, PyBytes, PyBytesRef, PyDictRef, PyIntRef, PyStrRef, PyTuple, PyTupleRef,
PyTypeRef,
};
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock, PyRwLockReadGuard,
PyRwLockWriteGuard,
},
};
use crate::{
anystr::{self, AnyStr},
builtins::PyType,
@@ -18,9 +11,17 @@ use crate::{
ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
},
common::{
atomic::{AtomicUsize, Ordering},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock,
PyRwLockReadGuard, PyRwLockWriteGuard,
},
},
function::{ArgBytesLike, ArgIterable, FuncArgs, IntoPyObject, OptionalArg, OptionalOption},
protocol::{
BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods,
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods,
},
sliceable::{PySliceableSequence, PySliceableSequenceMut, SequenceIndex},
types::{
@@ -29,10 +30,9 @@ use crate::{
},
utils::Either,
IdProtocol, PyClassDef, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef,
PyRef, PyResult, PyValue, TypeProtocol, VirtualMachine,
PyObjectView, PyObjectWrap, PyRef, PyResult, PyValue, TypeProtocol, VirtualMachine,
};
use bstr::ByteSlice;
use crossbeam_utils::atomic::AtomicCell;
use std::mem::size_of;
/// "bytearray(iterable_of_ints) -> bytearray\n\
@@ -50,7 +50,7 @@ use std::mem::size_of;
#[derive(Debug, Default)]
pub struct PyByteArray {
inner: PyRwLock<PyBytesInner>,
exports: AtomicCell<usize>,
exports: AtomicUsize,
}
pub type PyByteArrayRef = PyRef<PyByteArray>;
@@ -63,7 +63,7 @@ impl PyByteArray {
fn from_inner(inner: PyBytesInner) -> Self {
PyByteArray {
inner: PyRwLock::new(inner),
exports: AtomicCell::new(0),
exports: AtomicUsize::new(0),
}
}
@@ -118,6 +118,12 @@ impl PyByteArray {
Ok(())
}
#[cfg(debug_assertions)]
#[pyproperty]
fn exports(&self) -> usize {
self.exports.load(Ordering::Relaxed)
}
#[inline]
fn inner(&self) -> PyRwLockReadGuard<'_, PyBytesInner> {
self.inner.read()
@@ -708,36 +714,35 @@ impl Comparable for PyByteArray {
}
}
static BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| buffer.obj_as::<PyByteArray>().borrow_buf().into(),
obj_bytes_mut: |buffer| {
PyMappedRwLockWriteGuard::map(buffer.obj_as::<PyByteArray>().borrow_buf_mut(), |x| {
x.as_mut_slice()
})
.into()
},
release: |buffer| {
buffer
.obj_as::<PyByteArray>()
.exports
.fetch_sub(1, Ordering::Release);
},
retain: |buffer| {
buffer
.obj_as::<PyByteArray>()
.exports
.fetch_add(1, Ordering::Release);
},
};
impl AsBuffer for PyByteArray {
fn as_buffer(zelf: &crate::PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let buffer = PyBuffer::new(
zelf.as_object().to_owned(),
zelf.to_owned(),
BufferOptions {
readonly: false,
len: zelf.len(),
..Default::default()
},
);
Ok(buffer)
}
}
impl BufferInternal for PyRef<PyByteArray> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.borrow_buf().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
PyRwLockWriteGuard::map(self.inner_mut(), |inner| &mut *inner.elements).into()
}
fn release(&self) {
self.exports.fetch_sub(1);
}
fn retain(&self) {
self.exports.fetch_add(1);
fn as_buffer(zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
Ok(PyBuffer::new(
zelf.to_owned().into_object(),
BufferDescriptor::simple(zelf.len(), false),
&BUFFER_METHODS,
))
}
}
@@ -746,7 +751,7 @@ impl<'a> BufferResizeGuard<'a> for PyByteArray {
fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult<Self::Resizable> {
let w = self.inner.upgradable_read();
if self.exports.load() == 0 {
if self.exports.load(Ordering::SeqCst) == 0 {
Ok(parking_lot::lock_api::RwLockUpgradableReadGuard::upgrade(w))
} else {
Err(vm
vm/src/builtins/bytes.rs
+16 -29
View File
@@ -6,24 +6,20 @@ use crate::{
bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
},
common::hash::PyHash,
common::{hash::PyHash, lock::PyMutex},
function::{
ArgBytesLike, ArgIterable, IntoPyObject, IntoPyResult, OptionalArg, OptionalOption,
},
protocol::{BufferInternal, BufferOptions, PyBuffer, PyIterReturn, PyMappingMethods},
protocol::{BufferDescriptor, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods},
types::{
AsBuffer, AsMapping, Callable, Comparable, Constructor, Hashable, IterNext,
IterNextIterable, Iterable, PyComparisonOp, Unconstructible,
},
utils::Either,
IdProtocol, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef, PyRef, PyResult,
PyValue, TryFromBorrowedObject, TypeProtocol, VirtualMachine,
IdProtocol, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef, PyObjectView,
PyObjectWrap, PyRef, PyResult, PyValue, TryFromBorrowedObject, TypeProtocol, VirtualMachine,
};
use bstr::ByteSlice;
use rustpython_common::{
borrow::{BorrowedValue, BorrowedValueMut},
lock::PyMutex,
};
use std::mem::size_of;
use std::ops::Deref;
@@ -541,35 +537,26 @@ impl PyBytes {
}
}
static BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| buffer.obj_as::<PyBytes>().as_bytes().into(),
obj_bytes_mut: |_| panic!(),
release: |_| {},
retain: |_| {},
};
impl AsBuffer for PyBytes {
fn as_buffer(zelf: &crate::PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
fn as_buffer(zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let buf = PyBuffer::new(
zelf.as_object().to_owned(),
zelf.to_owned(),
BufferOptions {
len: zelf.len(),
..Default::default()
},
zelf.to_owned().into_object(),
BufferDescriptor::simple(zelf.len(), true),
&BUFFER_METHODS,
);
Ok(buf)
}
}
impl BufferInternal for PyRef<PyBytes> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.as_bytes().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
unreachable!("bytes is not mutable")
}
fn release(&self) {}
fn retain(&self) {}
}
impl AsMapping for PyBytes {
fn as_mapping(_zelf: &crate::PyObjectView<Self>, _vm: &VirtualMachine) -> PyMappingMethods {
fn as_mapping(_zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyMappingMethods {
PyMappingMethods {
length: Some(Self::length),
subscript: Some(Self::subscript),
vm/src/builtins/memory.rs
+726 -533
View File
File diff suppressed because it is too large. Load diff
vm/src/builtins/slice.rs
+74 -40
View File
@@ -9,6 +9,7 @@ use crate::{
use num_bigint::{BigInt, ToBigInt};
use num_traits::{One, Signed, ToPrimitive, Zero};
use std::ops::Range;
use std::option::Option;
#[pyclass(module = false, name = "slice")]
#[derive(Debug)]
@@ -287,53 +288,86 @@ impl SaturatedSlice {
// Equivalent to PySlice_AdjustIndices
/// Convert for usage in indexing the underlying rust collections. Called *after*
/// __index__ has been called on the Slice which might mutate the collection.
pub fn adjust_indices(&self, len: usize) -> (Range<usize>, Option<usize>, bool) {
// len should always be <= isize::MAX
let ilen = len.to_isize().unwrap_or(isize::MAX);
let (start, stop, step) = (self.start, self.stop, self.step);
let (start, stop, step, is_negative_step) = if step.is_negative() {
(
if stop == -1 {
ilen.saturating_add(1)
} else {
stop.saturating_add(1)
},
if start == -1 {
ilen
} else {
start.saturating_add(1)
},
step.saturating_abs(),
true,
)
} else {
(start, stop, step, false)
};
let step = step.to_usize();
let range = saturate_index(start, len)..saturate_index(stop, len);
let range = if range.start >= range.end {
range.start..range.start
} else {
// step overflow
if step.is_none() {
if is_negative_step {
(range.end - 1)..range.end
} else {
range.start..(range.start + 1)
}
pub fn adjust_indices(&self, len: usize) -> (Range<usize>, isize, usize) {
if len == 0 {
return (0..0, self.step, 0);
}
let range = if self.step.is_negative() {
let stop = if self.stop == -1 {
len
} else {
range
}
saturate_index(self.stop.saturating_add(1), len)
};
let start = if self.start == -1 {
len
} else {
saturate_index(self.start.saturating_add(1), len)
};
stop..start
} else {
saturate_index(self.start, len)..saturate_index(self.stop, len)
};
(range, step, is_negative_step)
let (range, slicelen) = if range.start >= range.end {
(range.start..range.start, 0)
} else {
let slicelen = (range.end - range.start - 1) / self.step.unsigned_abs() + 1;
(range, slicelen)
};
(range, self.step, slicelen)
}
pub fn iter(&self, len: usize) -> SaturatedSliceIter {
SaturatedSliceIter::new(self, len)
}
}
pub struct SaturatedSliceIter {
index: isize,
step: isize,
len: usize,
}
impl SaturatedSliceIter {
pub fn new(slice: &SaturatedSlice, seq_len: usize) -> Self {
let (range, step, len) = slice.adjust_indices(seq_len);
Self::from_adjust_indices(range, step, len)
}
pub fn from_adjust_indices(range: Range<usize>, step: isize, len: usize) -> Self {
let index = if step.is_negative() {
range.end as isize - 1
} else {
range.start as isize
};
Self { index, step, len }
}
pub fn positive_order(mut self) -> Self {
if self.step.is_negative() {
self.index += self.step * self.len.saturating_sub(1) as isize;
self.step = self.step.saturating_abs()
}
self
}
}
impl Iterator for SaturatedSliceIter {
type Item = usize;
fn next(&mut self) -> Option<Self::Item> {
if self.len == 0 {
return None;
}
self.len -= 1;
let ret = self.index as usize;
self.index += self.step;
Some(ret)
}
}
// Go from PyObjectRef to isize w/o overflow error, out of range values are substituted by
// isize::MIN or isize::MAX depending on type and value of step.
// Equivalent to PyNumber_AsSsize_t with err equal to None.
// Equivalent to PyEval_SliceIndex.
fn to_isize_index(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<isize>> {
if vm.is_none(obj) {
return Ok(None);
vm/src/cformat.rs
+1 -13
View File
@@ -392,19 +392,7 @@ impl CFormatSpec {
}
CFormatPreconversor::Str | CFormatPreconversor::Bytes => {
if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) {
let guard;
let vec;
let bytes = match buffer.as_contiguous() {
Some(bytes) => {
guard = bytes;
&*guard
}
None => {
vec = buffer.to_contiguous();
vec.as_slice()
}
};
Ok(self.format_bytes(bytes))
Ok(buffer.contiguous_or_collect(|bytes| self.format_bytes(bytes)))
} else {
let bytes = vm
.get_special_method(obj, "__bytes__")?
vm/src/function/buffer.rs
+9 -13
View File
@@ -40,7 +40,7 @@ impl PyObject {
impl ArgBytesLike {
pub fn borrow_buf(&self) -> BorrowedValue<'_, [u8]> {
self.0.as_contiguous().unwrap()
unsafe { self.0.contiguous_unchecked() }
}
pub fn with_ref<F, R>(&self, f: F) -> R
@@ -51,15 +51,11 @@ impl ArgBytesLike {
}
pub fn len(&self) -> usize {
self.borrow_buf().len()
self.0.desc.len
}
pub fn is_empty(&self) -> bool {
self.borrow_buf().is_empty()
}
pub fn to_cow(&self) -> std::borrow::Cow<[u8]> {
self.borrow_buf().to_vec().into()
self.len() == 0
}
}
@@ -72,7 +68,7 @@ impl From<ArgBytesLike> for PyBuffer {
impl TryFromBorrowedObject for ArgBytesLike {
fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Self> {
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
if buffer.options.contiguous {
if buffer.desc.is_contiguous() {
Ok(Self(buffer))
} else {
Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()))
@@ -86,7 +82,7 @@ pub struct ArgMemoryBuffer(PyBuffer);
impl ArgMemoryBuffer {
pub fn borrow_buf_mut(&self) -> BorrowedValueMut<'_, [u8]> {
self.0.as_contiguous_mut().unwrap()
unsafe { self.0.contiguous_mut_unchecked() }
}
pub fn with_ref<F, R>(&self, f: F) -> R
@@ -97,11 +93,11 @@ impl ArgMemoryBuffer {
}
pub fn len(&self) -> usize {
self.borrow_buf_mut().len()
self.0.desc.len
}
pub fn is_empty(&self) -> bool {
self.borrow_buf_mut().is_empty()
self.len() == 0
}
}
@@ -114,9 +110,9 @@ impl From<ArgMemoryBuffer> for PyBuffer {
impl TryFromBorrowedObject for ArgMemoryBuffer {
fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Self> {
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
if !buffer.options.contiguous {
if !buffer.desc.is_contiguous() {
Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()))
} else if buffer.options.readonly {
} else if buffer.desc.readonly {
Err(vm.new_type_error("buffer is not a read-write bytes-like object".to_owned()))
} else {
Ok(Self(buffer))
vm/src/protocol/buffer.rs
+399 -103
View File
@@ -1,139 +1,435 @@
//! Buffer protocol
//! https://docs.python.org/3/c-api/buffer.html
use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
use crate::common::rc::PyRc;
use crate::PyThreadingConstraint;
use crate::{PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TypeProtocol, VirtualMachine};
use std::{borrow::Cow, fmt::Debug};
use itertools::Itertools;
pub trait BufferInternal: Debug + PyThreadingConstraint {
/// Get the full inner buffer of this memory. You probably want `as_contiguous()`, as
/// `obj_bytes` doesn't take into account the range a memoryview might operate on, among other
/// footguns.
fn obj_bytes(&self) -> BorrowedValue<[u8]>;
/// Get the full inner buffer of this memory, mutably. You probably want
/// `as_contiguous_mut()`, as `obj_bytes` doesn't take into account the range a memoryview
/// might operate on, among other footguns.
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>;
fn release(&self);
// not included in PyBuffer protocol itself
fn retain(&self);
use crate::{
common::{
borrow::{BorrowedValue, BorrowedValueMut},
lock::{MapImmutable, PyMutex, PyMutexGuard},
},
sliceable::wrap_index,
types::{Constructor, Unconstructible},
PyObject, PyObjectPayload, PyObjectRef, PyObjectView, PyObjectWrap, PyRef, PyResult,
TryFromBorrowedObject, TypeProtocol, VirtualMachine,
};
use std::{borrow::Cow, fmt::Debug, ops::Range};
#[allow(clippy::type_complexity)]
pub struct BufferMethods {
pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<[u8]>,
pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<[u8]>,
pub release: fn(&PyBuffer),
pub retain: fn(&PyBuffer),
}
#[derive(Debug)]
pub struct PyBuffer {
pub obj: PyObjectRef,
pub options: BufferOptions,
pub(crate) internal: PyRc<dyn BufferInternal>,
}
impl PyBuffer {
pub fn new(
obj: PyObjectRef,
buffer: impl BufferInternal + 'static,
options: BufferOptions,
) -> Self {
buffer.retain();
Self {
obj,
options,
internal: PyRc::new(buffer),
}
}
pub fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(self.internal.obj_bytes())
}
pub fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(self.internal.obj_bytes_mut())
}
pub fn to_contiguous(&self) -> Vec<u8> {
self.internal.obj_bytes().to_vec()
}
pub fn clone_with_options(&self, options: BufferOptions) -> Self {
self.internal.retain();
Self {
obj: self.obj.clone(),
options,
internal: self.internal.clone(),
}
impl Debug for BufferMethods {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BufferMethods")
.field("obj_bytes", &(self.obj_bytes as usize))
.field("obj_bytes_mut", &(self.obj_bytes_mut as usize))
.field("release", &(self.release as usize))
.field("retain", &(self.retain as usize))
.finish()
}
}
#[derive(Debug, Clone)]
pub struct BufferOptions {
// buf
pub len: usize,
pub readonly: bool,
pub itemsize: usize,
pub format: Cow<'static, str>,
pub ndim: usize, // TODO: support multiple dimension array
pub shape: Vec<usize>,
pub strides: Vec<isize>,
// suboffsets
// RustPython fields
pub contiguous: bool,
pub struct PyBuffer {
pub obj: PyObjectRef,
pub desc: BufferDescriptor,
methods: &'static BufferMethods,
}
impl BufferOptions {
pub const DEFAULT: Self = BufferOptions {
len: 0,
readonly: true,
itemsize: 1,
format: Cow::Borrowed("B"),
ndim: 1,
shape: Vec::new(),
strides: Vec::new(),
contiguous: true,
};
}
impl PyBuffer {
pub fn new(obj: PyObjectRef, desc: BufferDescriptor, methods: &'static BufferMethods) -> Self {
let zelf = Self {
obj,
desc: desc.validate(),
methods,
};
zelf.retain();
zelf
}
impl Default for BufferOptions {
fn default() -> Self {
Self::DEFAULT
pub fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
self.desc
.is_contiguous()
.then(|| unsafe { self.contiguous_unchecked() })
}
pub fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
(!self.desc.readonly && self.desc.is_contiguous())
.then(|| unsafe { self.contiguous_mut_unchecked() })
}
/// # Safety
/// assume the buffer is contiguous
pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<[u8]> {
self.obj_bytes()
}
/// # Safety
/// assume the buffer is contiguous and writable
pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<[u8]> {
self.obj_bytes_mut()
}
pub fn append_to(&self, buf: &mut Vec<u8>) {
if let Some(bytes) = self.as_contiguous() {
buf.extend_from_slice(&bytes);
} else {
let bytes = &*self.obj_bytes();
self.desc.for_each_segment(true, |range| {
buf.extend_from_slice(&bytes[range.start as usize..range.end as usize])
});
}
}
pub fn contiguous_or_collect<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
let borrowed;
let mut collected;
let v = if let Some(bytes) = self.as_contiguous() {
borrowed = bytes;
&*borrowed
} else {
collected = vec![];
self.append_to(&mut collected);
&collected
};
f(v)
}
pub fn obj_as<T: PyObjectPayload>(&self) -> &PyObjectView<T> {
unsafe { self.obj.downcast_unchecked_ref() }
}
pub fn obj_bytes(&self) -> BorrowedValue<[u8]> {
(self.methods.obj_bytes)(self)
}
pub fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
(self.methods.obj_bytes_mut)(self)
}
pub fn release(&self) {
(self.methods.release)(self)
}
pub fn retain(&self) {
(self.methods.retain)(self)
}
// drop PyBuffer without calling release
// after this function, the owner should use forget()
// or wrap PyBuffer in the ManaullyDrop to prevent drop()
pub(crate) unsafe fn drop_without_release(&mut self) {
std::ptr::drop_in_place(&mut self.obj);
std::ptr::drop_in_place(&mut self.desc);
}
}
impl TryFromBorrowedObject for PyBuffer {
fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Self> {
let obj_cls = obj.class();
for cls in obj_cls.iter_mro() {
if let Some(f) = cls.slots.as_buffer.as_ref() {
return f(obj, vm);
}
let cls = obj.class();
if let Some(f) = cls.mro_find_map(|cls| cls.slots.as_buffer) {
return f(obj, vm);
}
Err(vm.new_type_error(format!(
"a bytes-like object is required, not '{}'",
obj_cls.name()
cls.name()
)))
}
}
// What we actually want to implement is:
// impl<T> Drop for T where T: BufferInternal
// but it is not supported by Rust
impl Drop for PyBuffer {
fn drop(&mut self) {
self.internal.release();
self.release();
}
}
impl Clone for PyBuffer {
fn clone(&self) -> Self {
self.clone_with_options(self.options.clone())
#[derive(Debug, Clone)]
pub struct BufferDescriptor {
/// product(shape) * itemsize
/// bytes length, but not the length for obj_bytes() even is contiguous
pub len: usize,
pub readonly: bool,
pub itemsize: usize,
pub format: Cow<'static, str>,
/// (shape, stride, suboffset) for each dimension
pub dim_desc: Vec<(usize, isize, isize)>,
// TODO: flags
}
impl BufferDescriptor {
pub fn simple(bytes_len: usize, readonly: bool) -> Self {
Self {
len: bytes_len,
readonly,
itemsize: 1,
format: Cow::Borrowed("B"),
dim_desc: vec![(bytes_len, 1, 0)],
}
}
pub fn format(
bytes_len: usize,
readonly: bool,
itemsize: usize,
format: Cow<'static, str>,
) -> Self {
Self {
len: bytes_len,
readonly,
itemsize,
format,
dim_desc: vec![(bytes_len / itemsize, itemsize as isize, 0)],
}
}
#[cfg(debug_assertions)]
pub fn validate(self) -> Self {
assert!(self.itemsize != 0);
assert!(self.ndim() != 0);
let mut shape_product = 1;
for (shape, stride, suboffset) in self.dim_desc.iter().cloned() {
shape_product *= shape;
assert!(suboffset >= 0);
assert!(stride != 0);
}
assert!(shape_product * self.itemsize == self.len);
self
}
#[cfg(not(debug_assertions))]
pub fn validate(self) -> Self {
self
}
pub fn ndim(&self) -> usize {
self.dim_desc.len()
}
pub fn is_contiguous(&self) -> bool {
if self.len == 0 {
return true;
}
let mut sd = self.itemsize;
for (shape, stride, _) in self.dim_desc.iter().cloned().rev() {
if shape > 1 && stride != sd as isize {
return false;
}
sd *= shape;
}
true
}
/// this function do not check the bound
/// panic if indices.len() != ndim
pub fn fast_position(&self, indices: &[usize]) -> isize {
let mut pos = 0;
for (i, (_, stride, suboffset)) in indices
.iter()
.cloned()
.zip_eq(self.dim_desc.iter().cloned())
{
pos += i as isize * stride + suboffset;
}
pos
}
/// panic if indices.len() != ndim
pub fn position(&self, indices: &[isize], vm: &VirtualMachine) -> PyResult<isize> {
let mut pos = 0;
for (i, (shape, stride, suboffset)) in indices
.iter()
.cloned()
.zip_eq(self.dim_desc.iter().cloned())
{
let i = wrap_index(i, shape).ok_or_else(|| {
vm.new_index_error(format!("index out of bounds on dimension {}", i))
})?;
pos += i as isize * stride + suboffset;
}
Ok(pos)
}
pub fn for_each_segment<F>(&self, try_conti: bool, mut f: F)
where
F: FnMut(Range<isize>),
{
if self.ndim() == 0 {
f(0..self.itemsize as isize);
return;
}
if try_conti && self.is_last_dim_contiguous() {
self._for_each_segment::<_, true>(0, 0, &mut f);
} else {
self._for_each_segment::<_, false>(0, 0, &mut f);
}
}
fn _for_each_segment<F, const CONTI: bool>(&self, mut index: isize, dim: usize, f: &mut F)
where
F: FnMut(Range<isize>),
{
let (shape, stride, suboffset) = self.dim_desc[dim];
if dim + 1 == self.ndim() {
if CONTI {
f(index..index + (shape * self.itemsize) as isize);
} else {
for _ in 0..shape {
let pos = index + suboffset;
f(pos..pos + self.itemsize as isize);
index += stride;
}
}
return;
}
for _ in 0..shape {
self._for_each_segment::<F, CONTI>(index + suboffset, dim + 1, f);
index += stride;
}
}
/// zip two BufferDescriptor with the same shape
pub fn zip_eq<F>(&self, other: &Self, try_conti: bool, mut f: F)
where
F: FnMut(Range<isize>, Range<isize>) -> bool,
{
if self.ndim() == 0 {
f(0..self.itemsize as isize, 0..other.itemsize as isize);
return;
}
if try_conti && self.is_last_dim_contiguous() {
self._zip_eq::<_, true>(other, 0, 0, 0, &mut f);
} else {
self._zip_eq::<_, false>(other, 0, 0, 0, &mut f);
}
}
fn _zip_eq<F, const CONTI: bool>(
&self,
other: &Self,
mut a_index: isize,
mut b_index: isize,
dim: usize,
f: &mut F,
) where
F: FnMut(Range<isize>, Range<isize>) -> bool,
{
let (shape, a_stride, a_suboffset) = self.dim_desc[dim];
let (_b_shape, b_stride, b_suboffset) = other.dim_desc[dim];
debug_assert_eq!(shape, _b_shape);
if dim + 1 == self.ndim() {
if CONTI {
if f(
a_index..a_index + (shape * self.itemsize) as isize,
b_index..b_index + (shape * other.itemsize) as isize,
) {
return;
}
} else {
for _ in 0..shape {
let a_pos = a_index + a_suboffset;
let b_pos = b_index + b_suboffset;
if f(
a_pos..a_pos + self.itemsize as isize,
b_pos..b_pos + other.itemsize as isize,
) {
return;
}
a_index += a_stride;
b_index += b_stride;
}
}
return;
}
for _ in 0..shape {
self._zip_eq::<F, CONTI>(
other,
a_index + a_suboffset,
b_index + b_suboffset,
dim + 1,
f,
);
a_index += a_stride;
b_index += b_stride;
}
}
fn is_last_dim_contiguous(&self) -> bool {
let (_, stride, suboffset) = self.dim_desc[self.ndim() - 1];
suboffset == 0 && stride == self.itemsize as isize
}
pub fn is_zero_in_shape(&self) -> bool {
for (shape, _, _) in self.dim_desc.iter().cloned() {
if shape == 0 {
return true;
}
}
false
}
// TODO: support fortain order
}
pub trait BufferResizeGuard<'a> {
type Resizable: 'a;
fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult<Self::Resizable>;
}
#[pyclass(module = false, name = "vec_buffer")]
#[derive(Debug, PyValue)]
pub struct VecBuffer {
data: PyMutex<Vec<u8>>,
}
#[pyimpl(flags(BASETYPE), with(Constructor))]
impl VecBuffer {
pub fn take(&self) -> Vec<u8> {
std::mem::take(&mut self.data.lock())
}
}
impl From<Vec<u8>> for VecBuffer {
fn from(data: Vec<u8>) -> Self {
Self {
data: PyMutex::new(data),
}
}
}
impl Unconstructible for VecBuffer {}
impl PyRef<VecBuffer> {
pub fn into_pybuffer(self, readonly: bool) -> PyBuffer {
let len = self.data.lock().len();
PyBuffer::new(
self.into_object(),
BufferDescriptor::simple(len, readonly),
&VEC_BUFFER_METHODS,
)
}
pub fn into_pybuffer_with_descriptor(self, desc: BufferDescriptor) -> PyBuffer {
PyBuffer::new(self.into_object(), desc, &VEC_BUFFER_METHODS)
}
}
static VEC_BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| {
PyMutexGuard::map_immutable(buffer.obj_as::<VecBuffer>().data.lock(), |x| x.as_slice())
.into()
},
obj_bytes_mut: |buffer| {
PyMutexGuard::map(buffer.obj_as::<VecBuffer>().data.lock(), |x| {
x.as_mut_slice()
})
.into()
},
release: |_| {},
retain: |_| {},
};
vm/src/protocol/mod.rs
+1 -1
View File
@@ -3,6 +3,6 @@ mod iter;
mod mapping;
mod object;
pub use buffer::{BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer};
pub use buffer::{BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer};
pub use iter::{PyIter, PyIterIter, PyIterReturn};
pub use mapping::{PyMapping, PyMappingMethods};
vm/src/sliceable.rs
+48 -134
View File
@@ -3,8 +3,8 @@ use std::ops::Range;
use crate::builtins::int::PyInt;
// export through slicable module, not slice.
use crate::builtins::slice::PySlice;
pub use crate::builtins::slice::{saturate_index, SaturatedSlice};
use crate::builtins::slice::{PySlice, SaturatedSliceIter};
use crate::utils::Either;
use crate::VirtualMachine;
use crate::{PyObjectRef, PyRef, PyResult, TypeProtocol};
@@ -28,55 +28,19 @@ pub trait PySliceableSequenceMut {
slice: SaturatedSlice,
items: &[Self::Item],
) -> PyResult<()> {
let (range, step, is_negative_step) = slice.adjust_indices(self.as_slice().len());
if !is_negative_step && step == Some(1) {
return if range.end - range.start == items.len() {
self.do_set_range(range, items);
Ok(())
} else {
Err(vm.new_buffer_error(
"Existing exports of data: object cannot be re-sized".to_owned(),
))
};
}
if let Some(step) = step {
let slicelen = if range.end > range.start {
(range.end - range.start - 1) / step + 1
} else {
0
};
if slicelen == items.len() {
let indexes = if is_negative_step {
itertools::Either::Left(range.rev().step_by(step))
} else {
itertools::Either::Right(range.step_by(step))
};
self.do_replace_indexes(indexes, items);
Ok(())
} else {
Err(vm.new_buffer_error(
"Existing exports of data: object cannot be re-sized".to_owned(),
))
}
let (range, step, slicelen) = slice.adjust_indices(self.as_slice().len());
if slicelen != items.len() {
Err(vm
.new_buffer_error("Existing exports of data: object cannot be re-sized".to_owned()))
} else if step == 1 {
self.do_set_range(range, items);
Ok(())
} else {
// edge case, step is too big for usize
// same behaviour as CPython
let slicelen = if range.start < range.end { 1 } else { 0 };
if match items.len() {
0 => slicelen == 0,
1 => {
self.do_set_range(range, items);
true
}
_ => false,
} {
Ok(())
} else {
Err(vm.new_buffer_error(
"Existing exports of data: object cannot be re-sized".to_owned(),
))
}
self.do_replace_indexes(
SaturatedSliceIter::from_adjust_indices(range, step, slicelen),
items,
);
Ok(())
}
}
@@ -86,82 +50,39 @@ pub trait PySliceableSequenceMut {
slice: SaturatedSlice,
items: &[Self::Item],
) -> PyResult<()> {
let (range, step, is_negative_step) = slice.adjust_indices(self.as_slice().len());
if !is_negative_step && step == Some(1) {
let (range, step, slicelen) = slice.adjust_indices(self.as_slice().len());
if step == 1 {
self.do_set_range(range, items);
return Ok(());
}
if let Some(step) = step {
let slicelen = if range.end > range.start {
(range.end - range.start - 1) / step + 1
} else {
0
};
if slicelen == items.len() {
let indexes = if is_negative_step {
itertools::Either::Left(range.rev().step_by(step))
} else {
itertools::Either::Right(range.step_by(step))
};
self.do_replace_indexes(indexes, items);
Ok(())
} else {
Err(vm.new_value_error(format!(
"attempt to assign sequence of size {} to extended slice of size {}",
items.len(),
slicelen
)))
}
Ok(())
} else if slicelen == items.len() {
self.do_replace_indexes(
SaturatedSliceIter::from_adjust_indices(range, step, slicelen),
items,
);
Ok(())
} else {
// edge case, step is too big for usize
// same behaviour as CPython
let slicelen = if range.start < range.end { 1 } else { 0 };
if match items.len() {
0 => slicelen == 0,
1 => {
self.do_set_range(range, items);
true
}
_ => false,
} {
Ok(())
} else {
Err(vm.new_value_error(format!(
"attempt to assign sequence of size {} to extended slice of size {}",
items.len(),
slicelen
)))
}
Err(vm.new_value_error(format!(
"attempt to assign sequence of size {} to extended slice of size {}",
items.len(),
slicelen
)))
}
}
fn delete_slice(&mut self, _vm: &VirtualMachine, slice: SaturatedSlice) -> PyResult<()> {
let (range, step, is_negative_step) = slice.adjust_indices(self.as_slice().len());
if range.start >= range.end {
return Ok(());
}
if !is_negative_step && step == Some(1) {
let (range, step, slicelen) = slice.adjust_indices(self.as_slice().len());
if slicelen == 0 {
Ok(())
} else if step == 1 {
self.do_delete_range(range);
return Ok(());
}
// step is not negative here
if let Some(step) = step {
let indexes = if is_negative_step {
itertools::Either::Left(range.clone().rev().step_by(step).rev())
} else {
itertools::Either::Right(range.clone().step_by(step))
};
self.do_delete_indexes(range, indexes);
Ok(())
} else {
// edge case, step is too big for usize
// same behaviour as CPython
self.do_delete_range(range);
self.do_delete_indexes(
range.clone(),
SaturatedSliceIter::from_adjust_indices(range, step, slicelen).positive_order(),
);
Ok(())
}
Ok(())
}
}
@@ -237,28 +158,21 @@ pub trait PySliceableSequence {
_vm: &VirtualMachine,
slice: SaturatedSlice,
) -> PyResult<Self::Sliced> {
let (range, step, is_negative_step) = slice.adjust_indices(self.len());
if range.start >= range.end {
return Ok(Self::empty());
}
if step == Some(1) {
return Ok(if is_negative_step {
let (range, step, slicelen) = slice.adjust_indices(self.len());
let sliced = if slicelen == 0 {
Self::empty()
} else if step == 1 {
if step.is_negative() {
self.do_slice_reverse(range)
} else {
self.do_slice(range)
});
}
if let Some(step) = step {
Ok(if is_negative_step {
self.do_stepped_slice_reverse(range, step)
} else {
self.do_stepped_slice(range, step)
})
}
} else if step.is_negative() {
self.do_stepped_slice_reverse(range, step.unsigned_abs())
} else {
Ok(self.do_slice(range))
}
self.do_stepped_slice(range, step.unsigned_abs())
};
Ok(sliced)
}
fn get_item(
vm/src/stdlib/builtins.rs
+4 -2
View File
@@ -1,7 +1,7 @@
//! Builtin function definitions.
//!
//! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html).
use crate::{PyObjectRef, VirtualMachine};
use crate::{PyClassImpl, PyObjectRef, VirtualMachine};
/// Built-in functions, exceptions, and other objects.
///
@@ -917,7 +917,9 @@ pub use builtins::{ascii, print};
pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) {
let ctx = &vm.ctx;
builtins::extend_module(vm, &module);
crate::protocol::VecBuffer::make_class(&vm.ctx);
let _ = builtins::extend_module(vm, &module);
let debug_mode: bool = vm.state.settings.optimize == 0;
extend_module!(vm, module, {
vm/src/stdlib/io.rs
+43 -92
View File
@@ -68,29 +68,27 @@ impl TryFromObject for Fildes {
mod _io {
use super::*;
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
lock::{
PyMappedThreadMutexGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard,
PyThreadMutex, PyThreadMutexGuard,
},
rc::PyRc,
};
use crate::{
builtins::{
PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyIntRef, PyMemoryView, PyStr,
PyStrRef, PyType, PyTypeRef,
},
common::lock::{
PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard,
PyThreadMutex, PyThreadMutexGuard,
},
function::{
ArgBytesLike, ArgIterable, ArgMemoryBuffer, FuncArgs, IntoPyObject, OptionalArg,
OptionalOption,
},
protocol::{BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer, PyIterReturn},
protocol::{
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, VecBuffer,
},
types::{Constructor, Destructor, IterNext, Iterable},
utils::Either,
vm::{ReprGuard, VirtualMachine},
IdProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, StaticType,
TryFromBorrowedObject, TryFromObject, TypeProtocol,
IdProtocol, PyContext, PyObject, PyObjectRef, PyObjectWrap, PyRef, PyResult, PyValue,
StaticType, TryFromBorrowedObject, TryFromObject, TypeProtocol,
};
use bstr::ByteSlice;
use crossbeam_utils::atomic::AtomicCell;
@@ -876,32 +874,20 @@ mod _io {
// TODO: loop if write() raises an interrupt
vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj,))?
} else {
let options = BufferOptions {
len,
..Default::default()
};
// TODO: see if we can encapsulate this pattern in a function in memory.rs like
// fn slice_as_memory<R>(s: &[u8], f: impl FnOnce(PyMemoryViewRef) -> R) -> R
let writebuf = PyRc::new(BufferedRawBuffer {
data: std::mem::take(&mut self.buffer).into(),
range: buf_range,
});
let raw = self.raw.as_ref().unwrap();
let memobj = PyMemoryView::from_buffer(
PyBuffer {
obj: raw.clone(),
options,
internal: writebuf.clone(),
},
let v = std::mem::take(&mut self.buffer);
let writebuf = VecBuffer::from(v).into_ref(vm);
let memobj = PyMemoryView::from_buffer_range(
writebuf.clone().into_pybuffer(true),
buf_range,
vm,
)?
.into_ref(vm);
// TODO: loop if write() raises an interrupt
let res = vm.call_method(raw, "write", (memobj.clone(),));
let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj.clone(),));
memobj.release();
self.buffer = std::mem::take(&mut writebuf.data.lock());
self.buffer = writebuf.take();
res?
};
@@ -1114,23 +1100,10 @@ mod _io {
let res = match v {
Either::A(v) => {
let v = v.unwrap_or(&mut self.buffer);
let options = BufferOptions {
len,
readonly: false,
..Default::default()
};
// TODO: see if we can encapsulate this pattern in a function in memory.rs like
// fn slice_as_memory<R>(s: &[u8], f: impl FnOnce(PyMemoryViewRef) -> R) -> R
let readbuf = PyRc::new(BufferedRawBuffer {
data: std::mem::take(v).into(),
range: buf_range,
});
let memobj = PyMemoryView::from_buffer(
PyBuffer {
obj: vm.ctx.none(),
options,
internal: readbuf.clone(),
},
let readbuf = VecBuffer::from(std::mem::take(v)).into_ref(vm);
let memobj = PyMemoryView::from_buffer_range(
readbuf.clone().into_pybuffer(false),
buf_range,
vm,
)?
.into_ref(vm);
@@ -1140,7 +1113,7 @@ mod _io {
vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj.clone(),));
memobj.release();
std::mem::swap(v, &mut readbuf.data.lock());
std::mem::swap(v, &mut readbuf.take());
res?
}
@@ -1319,29 +1292,6 @@ mod _io {
}
}
// this is a bit fancier than what CPython does, but in CPython if you store
// the memoryobj for the buffer until after the BufferedIO is destroyed, you
// can get a use-after-free, so this is a bit safe
#[derive(Debug)]
struct BufferedRawBuffer {
data: PyMutex<Vec<u8>>,
range: Range<usize>,
}
impl BufferInternal for BufferedRawBuffer {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
BorrowedValue::map(self.data.lock().into(), |data| &data[self.range.clone()])
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
BorrowedValueMut::map(self.data.lock().into(), |data| {
&mut data[self.range.clone()]
})
}
fn release(&self) {}
fn retain(&self) {}
}
pub fn get_offset(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Offset> {
let int = vm.to_index(&obj)?;
int.as_bigint().try_into().map_err(|_| {
@@ -3334,35 +3284,36 @@ mod _io {
#[pymethod]
fn getbuffer(self, vm: &VirtualMachine) -> PyResult<PyMemoryView> {
let options = BufferOptions {
readonly: false,
len: self.buffer.read().cursor.get_ref().len(),
..Default::default()
};
let buffer = PyBuffer::new(self.as_object().to_owned(), self, options);
let len = self.buffer.read().cursor.get_ref().len();
let buffer = PyBuffer::new(
self.into_object(),
BufferDescriptor::simple(len, false),
&BYTES_IO_BUFFER_METHODS,
);
let view = PyMemoryView::from_buffer(buffer, vm)?;
Ok(view)
}
}
impl BufferInternal for PyRef<BytesIO> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
PyRwLockReadGuard::map(self.buffer.read(), |x| x.cursor.get_ref().as_slice()).into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
PyRwLockWriteGuard::map(self.buffer.write(), |x| x.cursor.get_mut().as_mut_slice())
static BYTES_IO_BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| {
let zelf = buffer.obj_as::<BytesIO>();
PyRwLockReadGuard::map(zelf.buffer.read(), |x| x.cursor.get_ref().as_slice()).into()
},
obj_bytes_mut: |buffer| {
let zelf = buffer.obj_as::<BytesIO>();
PyRwLockWriteGuard::map(zelf.buffer.write(), |x| x.cursor.get_mut().as_mut_slice())
.into()
}
},
fn release(&self) {
self.exports.fetch_sub(1);
}
release: |buffer| {
buffer.obj_as::<BytesIO>().exports.fetch_sub(1);
},
fn retain(&self) {
self.exports.fetch_add(1);
}
}
retain: |buffer| {
buffer.obj_as::<BytesIO>().exports.fetch_add(1);
},
};
impl<'a> BufferResizeGuard<'a> for BytesIO {
type Resizable = PyRwLockWriteGuard<'a, BufferedIO>;
vm/src/stdlib/os.rs
+5 -1
View File
@@ -191,7 +191,11 @@ impl TryFromObject for PyPathLike {
fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
// path_converter in CPython
let obj = match PyBuffer::try_from_borrowed_object(vm, &obj) {
Ok(buffer) => PyBytes::from(buffer.internal.obj_bytes().to_vec()).into_pyobject(vm),
Ok(buffer) => {
let mut bytes = vec![];
buffer.append_to(&mut bytes);
PyBytes::from(bytes).into_pyobject(vm)
}
Err(_) => obj,
};
let fs_path = FsPath::try_from(obj, true, vm)?;
vm/src/stdlib/sre.rs
+6 -22
View File
@@ -140,31 +140,15 @@ mod _sre {
vm: &VirtualMachine,
f: F,
) -> PyResult<R> {
let buffer;
let guard;
let vec;
let s;
let str_drive = if self.isbytes {
buffer = PyBuffer::try_from_borrowed_object(vm, &string)?;
let bytes = match buffer.as_contiguous() {
Some(bytes) => {
guard = bytes;
&*guard
}
None => {
vec = buffer.to_contiguous();
vec.as_slice()
}
};
StrDrive::Bytes(bytes)
if self.isbytes {
let buffer = PyBuffer::try_from_borrowed_object(vm, &string)?;
buffer.contiguous_or_collect(|bytes| f(StrDrive::Bytes(bytes)))
} else {
s = string
let s = string
.payload::<PyStr>()
.ok_or_else(|| vm.new_type_error("expected string".to_owned()))?;
StrDrive::Str(s.as_str())
};
f(str_drive)
f(StrDrive::Str(s.as_str()))
}
}
fn with_state<R, F: FnOnce(State) -> PyResult<R>>(