diff --git a/Lib/io.py b/Lib/io.py index e37e81c2b..7362ae305 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -57,9 +57,7 @@ import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, - # XXX RUSTPYTHON TODO: IncrementalNewlineDecoder - # IncrementalNewlineDecoder, - text_encoding, TextIOWrapper) + IncrementalNewlineDecoder, text_encoding, TextIOWrapper) # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" @@ -100,10 +98,3 @@ except ImportError: pass else: RawIOBase.register(_WindowsConsoleIO) - - -# XXX: RUSTPYTHON; borrow IncrementalNewlineDecoder from _pyio -try: - from _pyio import IncrementalNewlineDecoder -except ImportError: - pass diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index ed41074f7..5592a2d5a 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -186,7 +186,6 @@ just fits in two lineS yup!! the end""" class TestSFpatches(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_html_diff(self): # Check SF patch 914575 for generating HTML differences f1a = ((patch914575_from1 + '123\n'*10)*3) diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index df894c5b2..bda6dee6b 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -280,8 +280,6 @@ class FileInputTests(BaseTests, unittest.TestCase): fi.readline() self.assertTrue(custom_open_hook.invoked, "openhook not invoked") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_readline(self): with open(TESTFN, 'wb') as f: f.write(b'A\nB\r\nC\r') diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 1ccd072b3..c4aba6919 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -84,7 +84,6 @@ class ImportTests(unittest.TestCase): with self.assertRaises(SyntaxError): imp.find_module('badsyntax_pep3120', path) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_issue1267(self): for mod, encoding, _ in self.test_strings: fp, filename, info = imp.find_module('module_' + mod, diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index a231ae1d5..603125f6d 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -701,8 +701,6 @@ class SourceLoaderTestHarness: class SourceOnlyLoaderTests(SourceLoaderTestHarness): """Test importlib.abc.SourceLoader for source-only loading.""" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_source(self): # Verify the source code is returned as a string. # If an OSError is raised by get_data then raise ImportError. @@ -761,8 +759,6 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): self.verify_module(module) self.assertFalse(hasattr(module, '__path__')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise # specified by an encoding line. @@ -882,8 +878,6 @@ class SourceLoaderGetSourceTests: """Tests for importlib.abc.SourceLoader.get_source().""" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_default_encoding(self): # Should have no problems with UTF-8 text. name = 'mod' @@ -893,8 +887,6 @@ class SourceLoaderGetSourceTests: returned_source = mock.get_source(name) self.assertEqual(returned_source, source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decoded_source(self): # Decoding should work. name = 'mod' @@ -905,8 +897,6 @@ class SourceLoaderGetSourceTests: returned_source = mock.get_source(name) self.assertEqual(returned_source, source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines(self): # PEP 302 says universal newlines should be used. name = 'mod' diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index dc27e4aa9..201f50691 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -36,22 +36,16 @@ class DecodeSourceBytesTests: source = "string ='ΓΌ'" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ut8_default(self): source_bytes = self.source.encode('utf-8') self.assertEqual(self.util.decode_source(source_bytes), self.source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_specified_encoding(self): source = '# coding=latin-1\n' + self.source source_bytes = source.encode('latin-1') assert source_bytes != source.encode('utf-8') self.assertEqual(self.util.decode_source(source_bytes), source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines(self): source = '\r\n'.join([self.source, self.source]) source_bytes = source.encode('utf-8') diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4f459675d..9cc22b959 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4121,41 +4121,6 @@ class CTextIOWrapperTest(TextIOWrapperTest): def test_newlines_input(self): super().test_newlines_input() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_read_one_by_one(self): - super().test_read_one_by_one() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_read_by_chunk(self): - super().test_read_by_chunk() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_1(self): - super().test_issue1395_1() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_2(self): - super().test_issue1395_2() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_3(self): - super().test_issue1395_3() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_4(self): - super().test_issue1395_4() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_5(self): - super().test_issue1395_5() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_reconfigure_write_through(self): @@ -4812,11 +4777,6 @@ class CMiscIOTest(MiscIOTest): name_of_module = "io", "_io" extra_exported = "BlockingIOError", - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test___all__(self): - super().test___all__() - def test_readinto_buffer_overflow(self): # Issue #18025 class BadReader(self.io.BufferedIOBase): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 706daf65b..72dd40136 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -187,8 +187,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(False, linecache.lazycache(FILENAME, None)) self.assertEqual(lines, linecache.getlines(FILENAME)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_smoke(self): lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) linecache.clearcache() @@ -199,8 +197,6 @@ class LineCacheTests(unittest.TestCase): # globals: this would error if the lazy value wasn't resolved. self.assertEqual(lines, linecache.getlines(NONEXISTENT_FILENAME)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_provide_after_failed_lookup(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) @@ -219,8 +215,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(False, linecache.lazycache('', globals())) self.assertEqual(False, linecache.lazycache('', globals())) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_already_cached(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) diff --git a/Lib/test/test_popen.py b/Lib/test/test_popen.py index e3030ee02..e6bfc480c 100644 --- a/Lib/test/test_popen.py +++ b/Lib/test/test_popen.py @@ -54,12 +54,10 @@ class PopenTest(unittest.TestCase): else: self.assertEqual(os.waitstatus_to_exitcode(status), 42) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_contextmanager(self): with os.popen("echo hello") as f: self.assertEqual(f.read(), "hello\n") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_iterating(self): with os.popen("echo hello") as f: self.assertEqual(list(f), ["hello\n"]) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index bc898567f..d7507eb7f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1056,8 +1056,6 @@ class ProcessTestCase(BaseTestCase): self.assertEqual(stdout, b"bananasplit") self.assertEqual(stderr, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines_and_text(self): args = [ sys.executable, "-c", @@ -1097,7 +1095,6 @@ class ProcessTestCase(BaseTestCase): self.assertEqual(p.stdout.read(), "line4\nline5\nline6\nline7\nline8") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_universal_newlines_communicate(self): # universal newlines through communicate() p = subprocess.Popen([sys.executable, "-c", @@ -1149,7 +1146,6 @@ class ProcessTestCase(BaseTestCase): p.communicate() self.assertEqual(p.returncode, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_universal_newlines_communicate_stdin_stdout_stderr(self): # universal newlines through communicate(), with stdin, stdout, stderr p = subprocess.Popen([sys.executable, "-c", @@ -3805,7 +3801,6 @@ class MiscTests(unittest.TestCase): raise KeyboardInterrupt # Test how __exit__ handles ^C. self._test_keyboardinterrupt_no_kill(popen_via_context_manager) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getoutput(self): self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 20f8085cc..28a869723 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1040,8 +1040,6 @@ class MiscTracebackCases(unittest.TestCase): class TestFrame(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basics(self): linecache.clearcache() linecache.lazycache("f", globals()) @@ -1059,8 +1057,6 @@ class TestFrame(unittest.TestCase): self.assertNotEqual(f, object()) self.assertEqual(f, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) @@ -1109,8 +1105,6 @@ class TestStack(unittest.TestCase): s = traceback.StackSummary.extract(traceback.walk_stack(None), limit=5) self.assertEqual(len(s), 5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_extract_stack_lookup_lines(self): linecache.clearcache() linecache.updatecache('/foo.py', globals()) @@ -1120,8 +1114,6 @@ class TestStack(unittest.TestCase): linecache.clearcache() self.assertEqual(s[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_extract_stackup_deferred_lookup_lines(self): linecache.clearcache() c = test_code('/foo.py', 'method') @@ -1153,8 +1145,6 @@ class TestStack(unittest.TestCase): [' File "foo.py", line 1, in fred\n line\n'], s.format()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) c = test_code('/foo.py', 'method') @@ -1162,8 +1152,6 @@ class TestStack(unittest.TestCase): s = traceback.StackSummary.extract(iter([(f, 6)]), capture_locals=True) self.assertEqual(s[0].locals, {'something': '1'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_locals(self): linecache.updatecache('/foo.py', globals()) c = test_code('/foo.py', 'method') @@ -1444,8 +1432,6 @@ class TestTracebackException(unittest.TestCase): traceback.walk_tb(exc_info[2]), limit=5) self.assertEqual(expected_stack, exc.stack) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lookup_lines(self): linecache.clearcache() e = Exception("uh oh") @@ -1457,8 +1443,6 @@ class TestTracebackException(unittest.TestCase): linecache.updatecache('/foo.py', globals()) self.assertEqual(exc.stack[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") @@ -1470,8 +1454,6 @@ class TestTracebackException(unittest.TestCase): self.assertEqual( exc.stack[0].locals, {'something': '1', 'other': "'string'"}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") diff --git a/common/src/lock/thread_mutex.rs b/common/src/lock/thread_mutex.rs index 1a5054264..93f77ca70 100644 --- a/common/src/lock/thread_mutex.rs +++ b/common/src/lock/thread_mutex.rs @@ -92,6 +92,11 @@ impl Default for ThreadMutex { Self::new(T::default()) } } +impl From for ThreadMutex { + fn from(val: T) -> Self { + Self::new(val) + } +} impl ThreadMutex { pub fn lock(&self) -> Option> { if self.raw.lock() { diff --git a/derive-impl/src/from_args.rs b/derive-impl/src/from_args.rs index 48bcd4fe2..7b5b68421 100644 --- a/derive-impl/src/from_args.rs +++ b/derive-impl/src/from_args.rs @@ -1,5 +1,6 @@ use proc_macro2::TokenStream; use quote::{quote, ToTokens}; +use syn::ext::IdentExt; use syn::{ parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Lit, Meta, NestedMeta, Result, }; @@ -138,7 +139,7 @@ fn generate_field((i, field): (usize, &Field)) -> Result { }; let name = field.ident.as_ref(); - let name_string = name.map(Ident::to_string); + let name_string = name.map(|ident| ident.unraw().to_string()); if matches!(&name_string, Some(s) if s.starts_with("_phantom")) { return Ok(quote! { #name: ::std::marker::PhantomData, diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index e9e85d1fe..9d1cc2f5c 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -56,11 +56,30 @@ impl IntoPyTuple for Vec { } } -macro_rules! impl_into_pyobj_tuple { - ($(($T:ident, $idx:tt)),+) => { +pub trait FromPyTuple<'a>: Sized { + fn from_pytuple(tuple: &'a PyTuple, vm: &VirtualMachine) -> PyResult; +} + +macro_rules! impl_from_into_pytuple { + ($($T:ident),+) => { impl<$($T: ToPyObject),*> IntoPyTuple for ($($T,)*) { fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref(vec![$(self.$idx.to_pyobject(vm)),*], &vm.ctx) + #[allow(non_snake_case)] + let ($($T,)*) = self; + PyTuple::new_ref(vec![$($T.to_pyobject(vm)),*], &vm.ctx) + } + } + + // TODO: figure out a way to let PyObjectRef implement TryFromBorrowedObject, and + // have this be a TryFromBorrowedObject bound + impl<'a, $($T: TryFromObject),*> FromPyTuple<'a> for ($($T,)*) { + fn from_pytuple(tuple: &'a PyTuple, vm: &VirtualMachine) -> PyResult { + #[allow(non_snake_case)] + let &[$(ref $T),+] = tuple.as_slice().try_into().map_err(|_| { + vm.new_type_error(format!("expected tuple with {} elements", impl_from_into_pytuple!(@count $($T)+))) + })?; + Ok(($($T::try_from_object(vm, $T.clone())?,)+)) + } } @@ -70,15 +89,21 @@ macro_rules! impl_into_pyobj_tuple { } } }; + (@count $($T:ident)+) => { + 0 $(+ impl_from_into_pytuple!(@discard $T))+ + }; + (@discard $T:ident) => { + 1 + }; } -impl_into_pyobj_tuple!((A, 0)); -impl_into_pyobj_tuple!((A, 0), (B, 1)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5), (G, 6)); +impl_from_into_pytuple!(A); +impl_from_into_pytuple!(A, B); +impl_from_into_pytuple!(A, B, C); +impl_from_into_pytuple!(A, B, C, D); +impl_from_into_pytuple!(A, B, C, D, E); +impl_from_into_pytuple!(A, B, C, D, E, F); +impl_from_into_pytuple!(A, B, C, D, E, F, G); impl PyTuple { pub(crate) fn fast_getitem(&self, idx: usize) -> PyObjectRef { @@ -185,6 +210,10 @@ impl PyTuple { Self { elements }.into_ref(&vm.ctx) }) } + + pub fn extract_tuple<'a, T: FromPyTuple<'a>>(&'a self, vm: &VirtualMachine) -> PyResult { + T::from_pytuple(self, vm) + } } #[pyclass( diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 38512d298..fb31ae487 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -120,7 +120,7 @@ mod _io { use crate::{ builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyIntRef, PyMemoryView, PyStr, - PyStrRef, PyType, PyTypeRef, + PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, }, class::StaticType, common::lock::{ @@ -148,6 +148,7 @@ mod _io { use malachite_bigint::{BigInt, BigUint}; use num_traits::ToPrimitive; use std::{ + borrow::Cow, io::{self, prelude::*, Cursor, SeekFrom}, ops::Range, }; @@ -2243,7 +2244,9 @@ mod _io { let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some(); let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?; - let (encoder, decoder) = Self::find_coder(&buffer, encoding.as_str(), &errors, vm)?; + let newline = args.newline.unwrap_or_default(); + let (encoder, decoder) = + Self::find_coder(&buffer, encoding.as_str(), &errors, newline, vm)?; *data = Some(TextIOData { buffer, @@ -2251,7 +2254,7 @@ mod _io { decoder, encoding, errors, - newline: args.newline.unwrap_or_default(), + newline, line_buffering: args.line_buffering.unwrap_or_default(), write_through: args.write_through.unwrap_or_default(), chunk_size: 8192, @@ -2291,6 +2294,7 @@ mod _io { buffer: &PyObject, encoding: &str, errors: &Py, + newline: Newlines, vm: &VirtualMachine, ) -> PyResult<( Option<(PyObjectRef, Option)>, @@ -2315,10 +2319,17 @@ mod _io { }; let decoder = if vm.call_method(buffer, "readable", ())?.try_to_bool(vm)? { - let incremental_decoder = - codec.get_incremental_decoder(Some(errors.to_owned()), vm)?; - // TODO: wrap in IncrementalNewlineDecoder if newlines == Universal | Passthrough - Some(incremental_decoder) + let decoder = codec.get_incremental_decoder(Some(errors.to_owned()), vm)?; + if let Newlines::Universal | Newlines::Passthrough = newline { + let args = IncrementalNewlineDecoderArgs { + decoder, + translate: matches!(newline, Newlines::Universal), + errors: None, + }; + Some(IncrementalNewlineDecoder::construct_and_init(args, vm)?.into()) + } else { + Some(decoder) + } } else { None }; @@ -2333,8 +2344,13 @@ mod _io { let mut data = self.data.lock().unwrap(); if let Some(data) = data.as_mut() { if let Some(encoding) = args.encoding { - let (encoder, decoder) = - Self::find_coder(&data.buffer, encoding.as_str(), &data.errors, vm)?; + let (encoder, decoder) = Self::find_coder( + &data.buffer, + encoding.as_str(), + &data.errors, + data.newline, + vm, + )?; data.encoding = encoding; data.encoder = encoder; data.decoder = decoder; @@ -2370,6 +2386,25 @@ mod _io { vm.call_method(&textio.buffer, "writable", ()) } + #[pygetset] + fn line_buffering(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.lock(vm)?.line_buffering) + } + + #[pygetset] + fn write_through(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.lock(vm)?.write_through) + } + + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyResult> { + let data = self.lock(vm)?; + let Some(decoder) = &data.decoder else { + return Ok(None); + }; + vm.get_attribute_opt(decoder.clone(), "newlines") + } + #[pygetset(name = "_CHUNK_SIZE")] fn chunksize(&self, vm: &VirtualMachine) -> PyResult { Ok(self.lock(vm)?.chunk_size) @@ -3122,6 +3157,229 @@ mod _io { } } + #[pyattr] + #[pyclass(name)] + #[derive(Debug, PyPayload, Default)] + struct IncrementalNewlineDecoder { + // TODO: Traverse + data: PyThreadMutex>, + } + + #[derive(Debug)] + struct IncrementalNewlineDecoderData { + decoder: PyObjectRef, + // afaict, this is used for nothing + // errors: PyObjectRef, + pendingcr: bool, + translate: bool, + seennl: SeenNewline, + } + + bitflags! { + #[derive(Debug, PartialEq, Eq, Copy, Clone)] + struct SeenNewline: u8 { + const LF = 1; + const CR = 2; + const CRLF = 4; + } + } + + impl DefaultConstructor for IncrementalNewlineDecoder {} + + #[derive(FromArgs)] + struct IncrementalNewlineDecoderArgs { + #[pyarg(any)] + decoder: PyObjectRef, + #[pyarg(any)] + translate: bool, + #[pyarg(any, default)] + errors: Option, + } + + impl Initializer for IncrementalNewlineDecoder { + type Args = IncrementalNewlineDecoderArgs; + fn init(zelf: PyRef, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let _ = args.errors; + let mut data = zelf.lock_opt(vm)?; + *data = Some(IncrementalNewlineDecoderData { + decoder: args.decoder, + translate: args.translate, + pendingcr: false, + seennl: SeenNewline::empty(), + }); + Ok(()) + } + } + + #[pyclass(with(Constructor, Initializer))] + impl IncrementalNewlineDecoder { + fn lock_opt( + &self, + vm: &VirtualMachine, + ) -> PyResult>> { + self.data + .lock() + .ok_or_else(|| vm.new_runtime_error("reentrant call inside nldecoder".to_owned())) + } + + fn lock( + &self, + vm: &VirtualMachine, + ) -> PyResult> { + let lock = self.lock_opt(vm)?; + PyThreadMutexGuard::try_map(lock, |x| x.as_mut()).map_err(|_| { + vm.new_value_error("I/O operation on uninitialized nldecoder".to_owned()) + }) + } + + #[pymethod] + fn decode(&self, args: NewlineDecodeArgs, vm: &VirtualMachine) -> PyResult { + self.lock(vm)?.decode(args.input, args.r#final, vm) + } + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, u64)> { + let data = self.lock(vm)?; + let (buffer, flag) = if vm.is_none(&data.decoder) { + (vm.ctx.new_bytes(vec![]).into(), 0) + } else { + vm.call_method(&data.decoder, "getstate", ())? + .try_to_ref::(vm)? + .extract_tuple::<(PyObjectRef, u64)>(vm)? + }; + let flag = (flag << 1) | (data.pendingcr as u64); + Ok((buffer, flag)) + } + + #[pymethod] + fn setstate(&self, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.lock(vm)?; + let (buffer, flag) = state.extract_tuple::<(PyObjectRef, u64)>(vm)?; + data.pendingcr = flag & 1 != 0; + if !vm.is_none(&data.decoder) { + vm.call_method(&data.decoder, "setstate", ((buffer, flag >> 1),))?; + } + Ok(()) + } + + #[pymethod] + fn reset(&self, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.lock(vm)?; + data.seennl = SeenNewline::empty(); + data.pendingcr = false; + if !vm.is_none(&data.decoder) { + vm.call_method(&data.decoder, "reset", ())?; + } + Ok(()) + } + + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyResult { + let data = self.lock(vm)?; + Ok(match data.seennl.bits() { + 1 => "\n".to_pyobject(vm), + 2 => "\r".to_pyobject(vm), + 3 => ("\r", "\n").to_pyobject(vm), + 4 => "\r\n".to_pyobject(vm), + 5 => ("\n", "\r\n").to_pyobject(vm), + 6 => ("\r", "\r\n").to_pyobject(vm), + 7 => ("\r", "\n", "\r\n").to_pyobject(vm), + _ => vm.ctx.none(), + }) + } + } + + #[derive(FromArgs)] + struct NewlineDecodeArgs { + #[pyarg(any)] + input: PyObjectRef, + #[pyarg(any, default)] + r#final: bool, + } + + impl IncrementalNewlineDecoderData { + fn decode( + &mut self, + input: PyObjectRef, + final_: bool, + vm: &VirtualMachine, + ) -> PyResult { + let output = if vm.is_none(&self.decoder) { + input + } else { + vm.call_method(&self.decoder, "decode", (input, final_))? + }; + let orig_output: PyStrRef = output.try_into_value(vm)?; + // this being Cow::Owned means we need to allocate a new string + let mut output = Cow::Borrowed(orig_output.as_str()); + if self.pendingcr && (final_ || !output.is_empty()) { + output = ["\r", &*output].concat().into(); + self.pendingcr = false; + } + if !final_ { + if let Some(s) = output.strip_suffix('\r') { + output = s.to_owned().into(); + self.pendingcr = true; + } + } + + if output.is_empty() { + return Ok(vm.ctx.empty_str.to_owned()); + } + + if (self.seennl == SeenNewline::LF || self.seennl.is_empty()) && !output.contains('\r') + { + if self.seennl.is_empty() && output.contains('\n') { + self.seennl.insert(SeenNewline::LF); + } + } else if !self.translate { + let mut matches = output.match_indices(['\r', '\n']); + while !self.seennl.is_all() { + let Some((i, c)) = matches.next() else { break }; + match c { + "\n" => self.seennl.insert(SeenNewline::LF), + // if c isn't \n, it can only be \r + _ if output[i + 1..].starts_with('\n') => { + matches.next(); + self.seennl.insert(SeenNewline::CRLF); + } + _ => self.seennl.insert(SeenNewline::CR), + } + } + } else { + let mut chunks = output.match_indices(['\r', '\n']); + let mut new_string = String::with_capacity(output.len()); + let mut last_modification_index = 0; + while let Some((cr_index, chunk)) = chunks.next() { + if chunk == "\r" { + // skip copying the CR + let mut next_chunk_index = cr_index + 1; + if output[cr_index + 1..].starts_with('\n') { + chunks.next(); + self.seennl.insert(SeenNewline::CRLF); + // skip the LF too + next_chunk_index += 1; + } else { + self.seennl.insert(SeenNewline::CR); + } + new_string.push_str(&output[last_modification_index..cr_index]); + new_string.push('\n'); + last_modification_index = next_chunk_index; + } else { + self.seennl.insert(SeenNewline::LF); + } + } + new_string.push_str(&output[last_modification_index..]); + output = new_string.into(); + } + + Ok(match output { + Cow::Borrowed(_) => orig_output, + Cow::Owned(s) => vm.ctx.new_str(s), + }) + } + } + #[pyattr] #[pyclass(name = "StringIO", base = "_TextIOBase")] #[derive(Debug, PyPayload)] diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index 7b904a32b..3bbf3fc81 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -782,7 +782,16 @@ pub trait Constructor: PyPayload { fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult; } -pub trait DefaultConstructor: PyPayload + Default {} +pub trait DefaultConstructor: PyPayload + Default { + fn construct_and_init(args: Self::Args, vm: &VirtualMachine) -> PyResult> + where + Self: Initializer, + { + let this = Self::default().into_ref(&vm.ctx); + Self::init(this.clone(), args, vm)?; + Ok(this) + } +} /// For types that cannot be instantiated through Python code. #[pyclass]