Upgrade test_io from 3.13.11 and fix more io tests (#6565)

* fix various test_io

* Upgrade test_io from 3.13.11

* Fix more test_io
This commit is contained in:
Jeong, YunWon authored and GitHub committed 2025-12-28 20:36:15 +09:00
1 parent 3600b6652d
commit 476b75de49
3 files changed
+249 -86

No files matched your search

+48 -51
View File
@@ -1818,10 +1818,6 @@ class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
support.gc_collect()
self.assertIsNone(wr(), wr)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_error_through_destructor(self):
return super().test_error_through_destructor()
def test_args_error(self):
# Issue #17275
with self.assertRaisesRegex(TypeError, "BufferedReader"):
@@ -1844,12 +1840,8 @@ class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
self.assertIsInstance(cm.exception.__cause__, TypeError)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_flush_error_on_close(self):
return super().test_flush_error_on_close()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_seek_character_device_file(self):
return super().test_seek_character_device_file()
def test_error_through_destructor(self):
return super().test_error_through_destructor()
def test_truncate_on_read_only(self):
return super().test_truncate_on_read_only()
@@ -2166,10 +2158,6 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
tp = io.BufferedWriter
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_error_through_destructor(self):
return super().test_error_through_destructor()
def test_initialization(self):
rawio = self.MockRawIO()
bufio = self.tp(rawio)
@@ -2205,8 +2193,8 @@ class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
self.tp(self.BytesIO(), 1024, 1024, 1024)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_flush_error_on_close(self):
return super().test_flush_error_on_close()
def test_error_through_destructor(self):
return super().test_error_through_destructor()
@unittest.skip('TODO: RUSTPYTHON; fallible allocation')
def test_constructor(self):
@@ -2692,10 +2680,6 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
tp = io.BufferedRandom
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_error_through_destructor(self):
return super().test_error_through_destructor()
@unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking')
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_garbage_collection(self):
@@ -2708,16 +2692,8 @@ class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
self.tp(self.BytesIO(), 1024, 1024, 1024)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_flush_error_on_close(self):
return super().test_flush_error_on_close()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_seek_character_device_file(self):
return super().test_seek_character_device_file()
@unittest.expectedFailure # TODO: RUSTPYTHON; f.read1(1) returns b'a'
def test_read1_after_write(self):
return super().test_read1_after_write()
def test_error_through_destructor(self):
return super().test_error_through_destructor()
@unittest.skip('TODO: RUSTPYTHON; fallible allocation')
def test_constructor(self):
@@ -3395,7 +3371,24 @@ class TextIOWrapperTest(unittest.TestCase):
self.assertEqual(f.tell(), p1)
f.close()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_tell_after_readline_with_cr(self):
# Test for gh-141314: TextIOWrapper.tell() assertion failure
# when dealing with standalone carriage returns
data = b'line1\r'
with self.open(os_helper.TESTFN, "wb") as f:
f.write(data)
with self.open(os_helper.TESTFN, "r") as f:
# Read line that ends with \r
line = f.readline()
self.assertEqual(line, "line1\n")
# This should not cause an assertion failure
pos = f.tell()
# Verify we can seek back to this position
f.seek(pos)
remaining = f.read()
self.assertEqual(remaining, "")
def test_seek_with_encoder_state(self):
f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004")
f.write("\u00e6\u0300")
@@ -4130,10 +4123,6 @@ class CTextIOWrapperTest(TextIOWrapperTest):
io = io
shutdown_error = "LookupError: unknown encoding: ascii"
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_error_through_destructor(self):
return super().test_error_through_destructor()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_initialization(self):
r = self.BytesIO(b"\xc3\xa9\n\n")
@@ -4291,6 +4280,10 @@ class CTextIOWrapperTest(TextIOWrapperTest):
def test_reconfigure_write_through(self):
return super().test_reconfigure_write_through()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_error_through_destructor(self):
return super().test_error_through_destructor()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_repr(self):
return super().test_repr()
@@ -4306,6 +4299,11 @@ class CTextIOWrapperTest(TextIOWrapperTest):
def test_pickling_subclass(self):
return super().test_pickling_subclass()
# TODO: RUSTPYTHON; euc_jis_2004 encoding not supported
@unittest.expectedFailure
def test_seek_with_encoder_state(self):
return super().test_seek_with_encoder_state()
class PyTextIOWrapperTest(TextIOWrapperTest):
io = pyio
@@ -4319,6 +4317,11 @@ class PyTextIOWrapperTest(TextIOWrapperTest):
def test_newlines(self):
return super().test_newlines()
# TODO: RUSTPYTHON; euc_jis_2004 encoding not supported
@unittest.expectedFailure
def test_seek_with_encoder_state(self):
return super().test_seek_with_encoder_state()
class IncrementalNewlineDecoderTest(unittest.TestCase):
@@ -4836,22 +4839,6 @@ class CMiscIOTest(MiscIOTest):
name_of_module = "io", "_io"
extra_exported = "BlockingIOError",
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferedWriter seeks on non-seekable pipe
def test_nonblock_pipe_write_bigbuf(self):
return super().test_nonblock_pipe_write_bigbuf()
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferedWriter seeks on non-seekable pipe
def test_nonblock_pipe_write_smallbuf(self):
return super().test_nonblock_pipe_write_smallbuf()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_warn_on_dealloc(self):
return super().test_warn_on_dealloc()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_warn_on_dealloc_fd(self):
return super().test_warn_on_dealloc_fd()
def test_readinto_buffer_overflow(self):
# Issue #18025
class BadReader(self.io.BufferedIOBase):
@@ -4916,6 +4903,16 @@ class CMiscIOTest(MiscIOTest):
def test_check_encoding_errors(self):
return super().test_check_encoding_errors()
# TODO: RUSTPYTHON; ResourceWarning not triggered by _io.FileIO
@unittest.expectedFailure
def test_warn_on_dealloc(self):
return super().test_warn_on_dealloc()
# TODO: RUSTPYTHON; ResourceWarning not triggered by _io.FileIO
@unittest.expectedFailure
def test_warn_on_dealloc_fd(self):
return super().test_warn_on_dealloc_fd()
class PyMiscIOTest(MiscIOTest):
io = pyio
+40 -7
View File
@@ -316,8 +316,44 @@ pub(crate) fn impl_pystruct_sequence_data(
// Generate try_from_elements trait override only when try_from_object=true
let try_from_elements_trait_override = if try_from_object {
let visible_field_idents: Vec<_> = visible_fields.iter().map(|f| &f.ident).collect();
let skipped_field_idents: Vec<_> = skipped_fields.iter().map(|f| &f.ident).collect();
let visible_field_inits: Vec<_> = visible_fields
.iter()
.map(|f| {
let ident = &f.ident;
let cfg_attrs = &f.cfg_attrs;
if cfg_attrs.is_empty() {
quote! { #ident: iter.next().unwrap().clone().try_into_value(vm)?, }
} else {
quote! {
#(#cfg_attrs)*
#ident: iter.next().unwrap().clone().try_into_value(vm)?,
}
}
})
.collect();
let skipped_field_inits: Vec<_> = skipped_fields
.iter()
.map(|f| {
let ident = &f.ident;
let cfg_attrs = &f.cfg_attrs;
if cfg_attrs.is_empty() {
quote! {
#ident: match iter.next() {
Some(v) => v.clone().try_into_value(vm)?,
None => vm.ctx.none(),
},
}
} else {
quote! {
#(#cfg_attrs)*
#ident: match iter.next() {
Some(v) => v.clone().try_into_value(vm)?,
None => vm.ctx.none(),
},
}
}
})
.collect();
quote! {
fn try_from_elements(
elements: Vec<::rustpython_vm::PyObjectRef>,
@@ -325,11 +361,8 @@ pub(crate) fn impl_pystruct_sequence_data(
) -> ::rustpython_vm::PyResult<Self> {
let mut iter = elements.into_iter();
Ok(Self {
#(#visible_field_idents: iter.next().unwrap().clone().try_into_value(vm)?,)*
#(#skipped_field_idents: match iter.next() {
Some(v) => v.clone().try_into_value(vm)?,
None => vm.ctx.none(),
},)*
#(#visible_field_inits)*
#(#skipped_field_inits)*
})
}
}
+161 -28
View File
@@ -191,6 +191,7 @@ mod _io {
borrow::Cow,
io::{self, Cursor, SeekFrom, prelude::*},
ops::Range,
sync::atomic::{AtomicBool, Ordering},
};
#[allow(clippy::let_and_return)]
@@ -935,7 +936,7 @@ mod _io {
let rewind = self.raw_offset() + (self.pos - self.write_pos);
if rewind != 0 {
self.raw_seek(-rewind, 1, vm)?;
self.raw_pos = -rewind;
self.raw_pos -= rewind;
}
while self.write_pos < self.write_end {
@@ -999,7 +1000,10 @@ mod _io {
};
if offset >= -self.pos && offset <= available {
self.pos += offset;
return Ok(current - available + offset);
// GH-95782: character devices may report raw position 0
// even after reading, which would make this negative
let result = current - available + offset;
return Ok(if result < 0 { 0 } else { result });
}
}
}
@@ -1111,9 +1115,51 @@ mod _io {
}
}
// TODO: something something check if error is BlockingIOError?
let _ = self.flush(vm);
// if BlockingIOError, shift buffer
// and try to buffer the new data; otherwise propagate the error
match self.flush(vm) {
Ok(()) => {}
Err(e) if e.fast_isinstance(vm.ctx.exceptions.blocking_io_error) => {
if self.readable() {
self.reset_read();
}
// Shift buffer and adjust positions
let shift = self.write_pos;
if shift > 0 {
self.buffer
.copy_within(shift as usize..self.write_end as usize, 0);
self.write_end -= shift;
self.raw_pos -= shift;
self.pos -= shift;
self.write_pos = 0;
}
let avail = self.buffer.len() - self.write_end as usize;
if buf_len <= avail {
// Everything can be buffered
let buf = obj.borrow_buf();
self.buffer[self.write_end as usize..][..buf_len].copy_from_slice(&buf);
self.write_end += buf_len as Offset;
self.pos += buf_len as Offset;
return Ok(buf_len);
}
// Buffer as much as possible and return BlockingIOError
let buf = obj.borrow_buf();
self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]);
self.write_end += avail as Offset;
self.pos += avail as Offset;
return Err(vm.invoke_exception(
vm.ctx.exceptions.blocking_io_error.to_owned(),
vec![
vm.new_pyobj(EAGAIN),
vm.new_pyobj("write could not complete without blocking"),
vm.new_pyobj(avail),
],
)?);
}
Err(e) => return Err(e),
}
// Only reach here if flush succeeded
let offset = self.raw_offset();
if offset != 0 {
self.raw_seek(-offset, 1, vm)?;
@@ -1556,6 +1602,7 @@ mod _io {
const SEEKABLE: bool = false;
fn data(&self) -> &PyThreadMutex<BufferedData>;
fn closing(&self) -> &AtomicBool;
fn lock(&self, vm: &VirtualMachine) -> PyResult<PyThreadMutexGuard<'_, BufferedData>> {
self.data()
@@ -1694,19 +1741,25 @@ mod _io {
Ok(self.lock(vm)?.raw.clone())
}
/// Get raw stream without holding the lock (for calling Python code safely)
fn get_raw_unlocked(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
let data = self.lock(vm)?;
Ok(data.check_init(vm)?.to_owned())
}
#[pygetset]
fn closed(&self, vm: &VirtualMachine) -> PyResult {
self.lock(vm)?.check_init(vm)?.get_attr("closed", vm)
self.get_raw_unlocked(vm)?.get_attr("closed", vm)
}
#[pygetset]
fn name(&self, vm: &VirtualMachine) -> PyResult {
self.lock(vm)?.check_init(vm)?.get_attr("name", vm)
self.get_raw_unlocked(vm)?.get_attr("name", vm)
}
#[pygetset]
fn mode(&self, vm: &VirtualMachine) -> PyResult {
self.lock(vm)?.check_init(vm)?.get_attr("mode", vm)
self.get_raw_unlocked(vm)?.get_attr("mode", vm)
}
#[pymethod]
@@ -1750,17 +1803,19 @@ mod _io {
#[pymethod]
fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult {
{
// Don't hold the lock while calling Python code to avoid reentrant lock issues
let raw = {
let data = zelf.lock(vm)?;
let raw = data.check_init(vm)?;
if file_closed(raw, vm)? {
return Ok(vm.ctx.none());
}
}
raw.to_owned()
};
// Set closing flag so that concurrent write() calls will fail
zelf.closing().store(true, Ordering::Release);
let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop);
let data = zelf.lock(vm)?;
let raw = data.raw.as_ref().unwrap();
let close_res = vm.call_method(raw, "close", ());
let close_res = vm.call_method(&raw, "close", ());
exception_chain(flush_res, close_res)
}
@@ -1774,10 +1829,9 @@ mod _io {
Self::WRITABLE
}
// TODO: this should be the default for an equivalent of _PyObject_GetState
#[pymethod]
fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())))
fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name())))
}
}
@@ -1830,6 +1884,10 @@ mod _io {
let n = std::cmp::min(have as usize, n);
return Ok(data.read_fast(n).unwrap());
}
// Flush write buffer before reading
if data.writable() {
data.flush_rewind(vm)?;
}
let mut v = vec![0; n];
data.reset_read();
let r = data
@@ -1855,6 +1913,19 @@ mod _io {
ensure_unclosed(raw, "readinto of closed file", vm)?;
data.readinto_generic(buf.into(), true, vm)
}
#[pymethod]
fn flush(&self, vm: &VirtualMachine) -> PyResult<()> {
// For read-only buffers, flush just calls raw.flush()
// Don't hold the lock while calling Python code to avoid reentrant lock issues
let raw = {
let data = self.reader().lock(vm)?;
data.check_init(vm)?.to_owned()
};
ensure_unclosed(&raw, "flush of closed file", vm)?;
vm.call_method(&raw, "flush", ())?;
Ok(())
}
}
fn exception_chain<T>(e1: PyResult<()>, e2: PyResult<T>) -> PyResult<T> {
@@ -1874,6 +1945,7 @@ mod _io {
struct BufferedReader {
_base: _BufferedIOBase,
data: PyThreadMutex<BufferedData>,
closing: AtomicBool,
}
impl BufferedMixin for BufferedReader {
@@ -1884,6 +1956,10 @@ mod _io {
fn data(&self) -> &PyThreadMutex<BufferedData> {
&self.data
}
fn closing(&self) -> &AtomicBool {
&self.closing
}
}
impl BufferedReadable for BufferedReader {
@@ -1922,6 +1998,27 @@ mod _io {
#[pymethod]
fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> {
// Check if close() is in progress (Issue #31976)
// If closing, wait for close() to complete by spinning until raw is closed.
// Note: This spin-wait has no timeout because close() is expected to always
// complete (flush + fd close).
if self.writer().closing().load(Ordering::Acquire) {
loop {
let raw = {
let data = self.writer().lock(vm)?;
match &data.raw {
Some(raw) => raw.to_owned(),
None => break, // detached
}
};
if file_closed(&raw, vm)? {
break;
}
// Yield to other threads
std::thread::yield_now();
}
return Err(vm.new_value_error("write to closed file".to_owned()));
}
let mut data = self.writer().lock(vm)?;
let raw = data.check_init(vm)?;
ensure_unclosed(raw, "write to closed file", vm)?;
@@ -1944,6 +2041,7 @@ mod _io {
struct BufferedWriter {
_base: _BufferedIOBase,
data: PyThreadMutex<BufferedData>,
closing: AtomicBool,
}
impl BufferedMixin for BufferedWriter {
@@ -1954,6 +2052,10 @@ mod _io {
fn data(&self) -> &PyThreadMutex<BufferedData> {
&self.data
}
fn closing(&self) -> &AtomicBool {
&self.closing
}
}
impl BufferedWritable for BufferedWriter {
@@ -1990,6 +2092,7 @@ mod _io {
struct BufferedRandom {
_base: _BufferedIOBase,
data: PyThreadMutex<BufferedData>,
closing: AtomicBool,
}
impl BufferedMixin for BufferedRandom {
@@ -2001,6 +2104,10 @@ mod _io {
fn data(&self) -> &PyThreadMutex<BufferedData> {
&self.data
}
fn closing(&self) -> &AtomicBool {
&self.closing
}
}
impl BufferedReadable for BufferedRandom {
@@ -3347,8 +3454,8 @@ mod _io {
}
#[pymethod]
fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())))
fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name())))
}
}
@@ -4908,7 +5015,7 @@ mod fileio {
zelf: &Py<Self>,
read_byte: OptionalSize,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
) -> PyResult<Option<Vec<u8>>> {
if !zelf.mode.load().contains(Mode::READABLE) {
return Err(new_unsupported_operation(
vm,
@@ -4926,6 +5033,10 @@ mod fileio {
vm.check_signals()?;
continue;
}
// Non-blocking mode: return None if EAGAIN
Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => {
return Ok(None);
}
Err(e) => return Err(Self::io_error(zelf, e, vm)),
}
};
@@ -4941,17 +5052,28 @@ mod fileio {
vm.check_signals()?;
continue;
}
// Non-blocking mode: return None if EAGAIN (only if no data read yet)
Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => {
if bytes.is_empty() {
return Ok(None);
}
break;
}
Err(e) => return Err(Self::io_error(zelf, e, vm)),
}
}
bytes
};
Ok(bytes)
Ok(Some(bytes))
}
#[pymethod]
fn readinto(zelf: &Py<Self>, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> {
fn readinto(
zelf: &Py<Self>,
obj: ArgMemoryBuffer,
vm: &VirtualMachine,
) -> PyResult<Option<usize>> {
if !zelf.mode.load().contains(Mode::READABLE) {
return Err(new_unsupported_operation(
vm,
@@ -4971,15 +5093,23 @@ mod fileio {
vm.check_signals()?;
continue;
}
// Non-blocking mode: return None if EAGAIN
Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => {
return Ok(None);
}
Err(e) => return Err(Self::io_error(zelf, e, vm)),
}
};
Ok(ret)
Ok(Some(ret))
}
#[pymethod]
fn write(zelf: &Py<Self>, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> {
fn write(
zelf: &Py<Self>,
obj: ArgBytesLike,
vm: &VirtualMachine,
) -> PyResult<Option<usize>> {
if !zelf.mode.load().contains(Mode::WRITABLE) {
return Err(new_unsupported_operation(
vm,
@@ -4989,12 +5119,15 @@ mod fileio {
let mut handle = zelf.get_fd(vm)?;
let len = obj
.with_ref(|b| handle.write(b))
.map_err(|err| Self::io_error(zelf, err, vm))?;
let len = match obj.with_ref(|b| handle.write(b)) {
Ok(n) => n,
// Non-blocking mode: return None if EAGAIN
Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => return Ok(None),
Err(e) => return Err(Self::io_error(zelf, e, vm)),
};
//return number of bytes written
Ok(len)
Ok(Some(len))
}
#[pymethod]
@@ -5060,8 +5193,8 @@ mod fileio {
}
#[pymethod]
fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())))
fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name())))
}
}