diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index d300337b9..13df61348 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -189,7 +189,6 @@ class ArrayReconstructorTest(unittest.TestCase): self.assertEqual(a, b, msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_unicode(self): teststr = "Bonne Journ\xe9e \U0002030a\U00020347" testcases = ( @@ -1180,7 +1179,6 @@ class BaseTest: basesize = support.calcvobjsize('Pn2Pi') support.check_sizeof(self, a, basesize) - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_initialize_with_unicode(self): if self.typecode not in ('u', 'w'): with self.assertRaises(TypeError) as cm: @@ -1267,207 +1265,11 @@ class UnicodeTest(StringTest, unittest.TestCase): self.assertEqual(len(a), 0) self.assertEqual(a.typecode, 'u') - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_add(self): - return super().test_add() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extend(self): - return super().test_extend() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_iadd(self): - return super().test_iadd() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setiadd(self): - return super().test_setiadd() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setslice(self): - return super().test_setslice() - class UCS4Test(UnicodeTest): typecode = 'w' minitemsize = 4 - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_buffer(self): - return super().test_buffer() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_buffer_info(self): - return super().test_buffer_info() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_byteswap(self): - return super().test_byteswap() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_clear(self): - return super().test_clear() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_cmp(self): - return super().test_cmp() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_constructor(self): - return super().test_constructor() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_constructor_with_iterable_argument(self): - return super().test_constructor_with_iterable_argument() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_copy(self): - return super().test_copy() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_count(self): - return super().test_count() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_coveritertraverse(self): - return super().test_coveritertraverse() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_deepcopy(self): - return super().test_deepcopy() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_delitem(self): - return super().test_delitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_exhausted_iterator(self): - return super().test_exhausted_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_exhausted_reverse_iterator(self): - return super().test_exhausted_reverse_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extended_getslice(self): - return super().test_extended_getslice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extended_set_del_slice(self): - return super().test_extended_set_del_slice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_filewrite(self): - return super().test_filewrite() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_fromarray(self): - return super().test_fromarray() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_fromfile_ioerror(self): - return super().test_fromfile_ioerror() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_getitem(self): - return super().test_getitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_getslice(self): - return super().test_getslice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_imul(self): - return super().test_imul() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_index(self): - return super().test_index() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_insert(self): - return super().test_insert() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_issue17223(self): - return super().test_issue17223() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_iterator_pickle(self): - return super().test_iterator_pickle() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_len(self): - return super().test_len() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_mul(self): - return super().test_mul() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pickle(self): - return super().test_pickle() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pickle_for_empty_array(self): - return super().test_pickle_for_empty_array() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pop(self): - return super().test_pop() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reduce_ex(self): - return super().test_reduce_ex() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_remove(self): - return super().test_remove() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_repr(self): - return super().test_repr() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse(self): - return super().test_reverse() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse_iterator(self): - return super().test_reverse_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse_iterator_picking(self): - return super().test_reverse_iterator_picking() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setitem(self): - return super().test_setitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_str(self): - return super().test_str() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofrombytes(self): - return super().test_tofrombytes() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofromfile(self): - return super().test_tofromfile() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofromlist(self): - return super().test_tofromlist() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_unicode(self): - return super().test_unicode() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_weakref(self): - return super().test_weakref() - class NumberTest(BaseTest): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6e00f8eb1..86f1cf9bc 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1137,7 +1137,6 @@ class TestArrayWrites(unittest.TestCase): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_char_write(self): import array, string a = array.array('w', string.ascii_letters) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index eef630754..1c52eb0dc 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1754,7 +1754,6 @@ class ReTests(unittest.TestCase): for x in not_decimal_digits: self.assertIsNone(re.match(r'^\d$', x)) - @unittest.expectedFailure # TODO: RUSTPYTHON; a = array.array(typecode)\n ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d) @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') def test_empty_array(self): # SF buf 1647541 diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 19f6c48f2..f2a16d723 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -60,7 +60,7 @@ pub mod array { match c { $($c => Ok(ArrayContentType::$n(Vec::new())),)* _ => Err( - "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)".into() + "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)".into() ), } } @@ -473,6 +473,7 @@ pub mod array { (SignedByte, i8, 'b', "b"), (UnsignedByte, u8, 'B', "B"), (PyUnicode, WideChar, 'u', "u"), + (PyUcs4, Ucs4Char, 'w', "w"), (SignedShort, raw::c_short, 'h', "h"), (UnsignedShort, raw::c_ushort, 'H', "H"), (SignedInt, raw::c_int, 'i', "i"), @@ -488,6 +489,11 @@ pub mod array { #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] pub struct WideChar(wchar_t); + /// Element type for the 'w' typecode: always a 4-byte unicode code point + /// (Py_UCS4), unlike 'u' which is platform-dependent `wchar_t`. + #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] + pub struct Ucs4Char(u32); + trait ArrayElement: Sized { fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult; fn byteswap(self) -> Self; @@ -574,6 +580,47 @@ pub mod array { } } + impl ArrayElement for Ucs4Char { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let s = obj.downcast::().map_err(|obj| { + vm.new_type_error(format!( + "array item must be a unicode character, not {}", + obj.class().name() + )) + })?; + s.as_wtf8() + .code_points() + .exactly_one() + .map(|ch| Self(ch.to_u32())) + .map_err(|e| { + vm.new_type_error(format!( + "array item must be a unicode character, not a string of length {}", + e.count() + )) + }) + } + fn byteswap(self) -> Self { + Self(self.0.swap_bytes()) + } + fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef { + unreachable!() + } + } + + impl ToPyResult for Ucs4Char { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + Ok(u32_to_char(self.0) + .map_err(|msg| vm.new_value_error(msg))? + .to_pyobject(vm)) + } + } + + impl fmt::Display for Ucs4Char { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + unreachable!("`repr(array('w'))` calls `PyStr::repr`") + } + } + fn u32_to_char(ch: u32) -> Result { CodePoint::from_u32(ch) .ok_or_else(|| format!("character U+{ch:4x} is not in range [U+0000; U+10ffff]")) @@ -663,7 +710,7 @@ pub mod array { let value = init.read().typecode(); match (spec, value) { (spec, ch) if spec == ch => array.frombytes(&init.get_bytes()), - (spec, 'u') => { + (spec, 'u' | 'w') if !matches!(spec, 'u' | 'w') => { return Err(vm.new_type_error(format!( "cannot use a unicode array to initialize an array with typecode '{spec}'" ))) @@ -675,7 +722,7 @@ pub mod array { } } } else if let Some(wtf8) = init.downcast_ref::() { - if spec == 'u' { + if matches!(spec, 'u' | 'w') { let bytes = Self::_unicode_to_wchar_bytes(wtf8.as_wtf8(), array.itemsize()); array.frombytes_move(bytes); } else { @@ -824,10 +871,10 @@ pub mod array { obj.class().name() )) })?; - if zelf.read().typecode() != 'u' { - return Err( - vm.new_value_error("fromunicode() may only be called on unicode type arrays") - ); + if !matches!(zelf.read().typecode(), 'u' | 'w') { + return Err(vm.new_value_error( + "fromunicode() may only be called on unicode type arrays ('u' or 'w')", + )); } let mut w = zelf.try_resizable(vm)?; let bytes = Self::_unicode_to_wchar_bytes(wtf8, w.itemsize()); @@ -838,10 +885,10 @@ pub mod array { #[pymethod] fn tounicode(&self, vm: &VirtualMachine) -> PyResult { let array = self.array.read(); - if array.typecode() != 'u' { - return Err( - vm.new_value_error("tounicode() may only be called on unicode type arrays") - ); + if !matches!(array.typecode(), 'u' | 'w') { + return Err(vm.new_value_error( + "tounicode() may only be called on unicode type arrays ('u' or 'w')", + )); } let bytes = array.get_bytes(); Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm) @@ -1152,7 +1199,7 @@ pub mod array { let array = zelf.read(); let cls = zelf.class().to_owned(); let typecode = vm.ctx.new_str(array.typecode_str()); - let values = if array.typecode() == 'u' { + let values = if matches!(array.typecode(), 'u' | 'w') { let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?; s.code_points().map(|x| x.to_pyobject(vm)).collect() } else { @@ -1266,13 +1313,14 @@ pub mod array { fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { let class = zelf.class(); let class_name = class.name(); - if zelf.read().typecode() == 'u' { + let typecode = zelf.read().typecode(); + if matches!(typecode, 'u' | 'w') { if zelf.__len__() == 0 { - return Ok(format!("{class_name}('u')")); + return Ok(format!("{class_name}('{typecode}')")); } let to_unicode = zelf.tounicode(vm)?; let escape = crate::vm::literal::escape::UnicodeEscape::new_repr(&to_unicode); - return Ok(format!("{}('u', {})", class_name, escape.str_repr())); + return Ok(format!("{class_name}('{typecode}', {})", escape.str_repr())); } zelf.read().repr(&class_name, vm) } @@ -1524,6 +1572,7 @@ pub mod array { _ => None, }; } + 'w' => return Some(Self::Utf32 { big_endian }), 'f' => { // Copied from CPython const Y: f32 = 16711938.0; diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index a174e2fd6..84c0ac17f 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -85,6 +85,7 @@ pub(crate) enum FormatType { UByte = b'B', Char = b'c', WideChar = b'u', + Ucs4Char = b'w', Str = b's', Pascal = b'p', Short = b'h', @@ -189,6 +190,7 @@ impl FormatType { unpack: Some(unpack_char), }, Self::WideChar => native_info!(wchar_t), + Self::Ucs4Char => native_info!(u32), Self::Short => native_info!(raw::c_short), Self::UShort => native_info!(raw::c_ushort), Self::Int => native_info!(raw::c_int), @@ -344,9 +346,10 @@ impl FormatCode { let code = FormatType::try_from(c) .ok() .filter(|c| match c { - FormatType::SSizeT | FormatType::SizeT | FormatType::VoidP => { - endianness == Endianness::Native - } + FormatType::SSizeT + | FormatType::SizeT + | FormatType::VoidP + | FormatType::Ucs4Char => endianness == Endianness::Native, _ => true, }) .ok_or_else(|| "bad char in struct format".to_owned())?;