From 76ca1ff106201c5c385a6465e2bf94f59692245e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=95=ED=9A=A8=EC=A2=85?= Date: Wed, 15 Jul 2026 23:18:03 +0900 Subject: [PATCH] Add 'w' (Py_UCS4) typecode support to array (#8266) * Support array 'w' (Py_UCS4) typecode Python 3.13 added the 'w' typecode holding fixed 4-byte unicode code points, unlike 'u' which is platform-dependent wchar_t. Add a Ucs4Char element type and route it through the existing unicode-array paths: constructor from str, fromunicode()/tounicode(), __reduce_ex__, repr, byteswap, and the machine-format code (Utf32). Also register 'w' as a struct format char in vm::buffer. Remove the now-passing expectedFailure markers for 'w' tests in test_array.py. Assisted-by: Claude * Reject non-native byte order for the 'w' format char FormatType::parse accepted 'w' after '<', '>', '!', or '=', but the non-native branch of FormatType::info has no Ucs4Char arm and hits unreachable!(), so struct.calcsize('>w') panicked the interpreter. Add Ucs4Char to the existing native-only filter in FormatCode::parse (alongside SSizeT/SizeT/VoidP) so those formats fail with the regular "bad char in struct format" error instead. Native 'w' and memoryview(array('w', ...)) are unaffected. Assisted-by: Claude * Unmark 'w'-typecode tests in test_re and test_csv test_re's test_empty_array and test_csv's test_char_write construct array('w', ...) and were marked expectedFailure while the typecode was unsupported. Now that 'w' works they pass, and regrtest counts the unexpected successes as failures in CI. Remove the two markers, which restores both tests to their CPython v3.14.6 form; test_re, test_csv, and test_array all pass. Assisted-by: Claude --- Lib/test/test_array.py | 198 ------------------------------------- Lib/test/test_csv.py | 1 - Lib/test/test_re.py | 1 - crates/stdlib/src/array.rs | 79 ++++++++++++--- crates/vm/src/buffer.rs | 9 +- 5 files changed, 70 insertions(+), 218 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index d300337b9..13df61348 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -189,7 +189,6 @@ class ArrayReconstructorTest(unittest.TestCase): self.assertEqual(a, b, msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_unicode(self): teststr = "Bonne Journ\xe9e \U0002030a\U00020347" testcases = ( @@ -1180,7 +1179,6 @@ class BaseTest: basesize = support.calcvobjsize('Pn2Pi') support.check_sizeof(self, a, basesize) - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_initialize_with_unicode(self): if self.typecode not in ('u', 'w'): with self.assertRaises(TypeError) as cm: @@ -1267,207 +1265,11 @@ class UnicodeTest(StringTest, unittest.TestCase): self.assertEqual(len(a), 0) self.assertEqual(a.typecode, 'u') - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_add(self): - return super().test_add() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extend(self): - return super().test_extend() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_iadd(self): - return super().test_iadd() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setiadd(self): - return super().test_setiadd() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setslice(self): - return super().test_setslice() - class UCS4Test(UnicodeTest): typecode = 'w' minitemsize = 4 - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_buffer(self): - return super().test_buffer() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_buffer_info(self): - return super().test_buffer_info() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_byteswap(self): - return super().test_byteswap() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_clear(self): - return super().test_clear() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_cmp(self): - return super().test_cmp() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_constructor(self): - return super().test_constructor() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_constructor_with_iterable_argument(self): - return super().test_constructor_with_iterable_argument() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_copy(self): - return super().test_copy() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_count(self): - return super().test_count() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_coveritertraverse(self): - return super().test_coveritertraverse() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_deepcopy(self): - return super().test_deepcopy() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_delitem(self): - return super().test_delitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_exhausted_iterator(self): - return super().test_exhausted_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_exhausted_reverse_iterator(self): - return super().test_exhausted_reverse_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extended_getslice(self): - return super().test_extended_getslice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_extended_set_del_slice(self): - return super().test_extended_set_del_slice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_filewrite(self): - return super().test_filewrite() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_fromarray(self): - return super().test_fromarray() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_fromfile_ioerror(self): - return super().test_fromfile_ioerror() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_getitem(self): - return super().test_getitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_getslice(self): - return super().test_getslice() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_imul(self): - return super().test_imul() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_index(self): - return super().test_index() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_insert(self): - return super().test_insert() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_issue17223(self): - return super().test_issue17223() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_iterator_pickle(self): - return super().test_iterator_pickle() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_len(self): - return super().test_len() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_mul(self): - return super().test_mul() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pickle(self): - return super().test_pickle() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pickle_for_empty_array(self): - return super().test_pickle_for_empty_array() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_pop(self): - return super().test_pop() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reduce_ex(self): - return super().test_reduce_ex() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_remove(self): - return super().test_remove() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_repr(self): - return super().test_repr() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse(self): - return super().test_reverse() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse_iterator(self): - return super().test_reverse_iterator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_reverse_iterator_picking(self): - return super().test_reverse_iterator_picking() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_setitem(self): - return super().test_setitem() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_str(self): - return super().test_str() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofrombytes(self): - return super().test_tofrombytes() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofromfile(self): - return super().test_tofromfile() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_tofromlist(self): - return super().test_tofromlist() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_unicode(self): - return super().test_unicode() - - @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' - def test_weakref(self): - return super().test_weakref() - class NumberTest(BaseTest): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6e00f8eb1..86f1cf9bc 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1137,7 +1137,6 @@ class TestArrayWrites(unittest.TestCase): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_char_write(self): import array, string a = array.array('w', string.ascii_letters) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index eef630754..1c52eb0dc 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1754,7 +1754,6 @@ class ReTests(unittest.TestCase): for x in not_decimal_digits: self.assertIsNone(re.match(r'^\d$', x)) - @unittest.expectedFailure # TODO: RUSTPYTHON; a = array.array(typecode)\n ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d) @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') def test_empty_array(self): # SF buf 1647541 diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 19f6c48f2..f2a16d723 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -60,7 +60,7 @@ pub mod array { match c { $($c => Ok(ArrayContentType::$n(Vec::new())),)* _ => Err( - "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)".into() + "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)".into() ), } } @@ -473,6 +473,7 @@ pub mod array { (SignedByte, i8, 'b', "b"), (UnsignedByte, u8, 'B', "B"), (PyUnicode, WideChar, 'u', "u"), + (PyUcs4, Ucs4Char, 'w', "w"), (SignedShort, raw::c_short, 'h', "h"), (UnsignedShort, raw::c_ushort, 'H', "H"), (SignedInt, raw::c_int, 'i', "i"), @@ -488,6 +489,11 @@ pub mod array { #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] pub struct WideChar(wchar_t); + /// Element type for the 'w' typecode: always a 4-byte unicode code point + /// (Py_UCS4), unlike 'u' which is platform-dependent `wchar_t`. + #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] + pub struct Ucs4Char(u32); + trait ArrayElement: Sized { fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult; fn byteswap(self) -> Self; @@ -574,6 +580,47 @@ pub mod array { } } + impl ArrayElement for Ucs4Char { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let s = obj.downcast::().map_err(|obj| { + vm.new_type_error(format!( + "array item must be a unicode character, not {}", + obj.class().name() + )) + })?; + s.as_wtf8() + .code_points() + .exactly_one() + .map(|ch| Self(ch.to_u32())) + .map_err(|e| { + vm.new_type_error(format!( + "array item must be a unicode character, not a string of length {}", + e.count() + )) + }) + } + fn byteswap(self) -> Self { + Self(self.0.swap_bytes()) + } + fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef { + unreachable!() + } + } + + impl ToPyResult for Ucs4Char { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + Ok(u32_to_char(self.0) + .map_err(|msg| vm.new_value_error(msg))? + .to_pyobject(vm)) + } + } + + impl fmt::Display for Ucs4Char { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + unreachable!("`repr(array('w'))` calls `PyStr::repr`") + } + } + fn u32_to_char(ch: u32) -> Result { CodePoint::from_u32(ch) .ok_or_else(|| format!("character U+{ch:4x} is not in range [U+0000; U+10ffff]")) @@ -663,7 +710,7 @@ pub mod array { let value = init.read().typecode(); match (spec, value) { (spec, ch) if spec == ch => array.frombytes(&init.get_bytes()), - (spec, 'u') => { + (spec, 'u' | 'w') if !matches!(spec, 'u' | 'w') => { return Err(vm.new_type_error(format!( "cannot use a unicode array to initialize an array with typecode '{spec}'" ))) @@ -675,7 +722,7 @@ pub mod array { } } } else if let Some(wtf8) = init.downcast_ref::() { - if spec == 'u' { + if matches!(spec, 'u' | 'w') { let bytes = Self::_unicode_to_wchar_bytes(wtf8.as_wtf8(), array.itemsize()); array.frombytes_move(bytes); } else { @@ -824,10 +871,10 @@ pub mod array { obj.class().name() )) })?; - if zelf.read().typecode() != 'u' { - return Err( - vm.new_value_error("fromunicode() may only be called on unicode type arrays") - ); + if !matches!(zelf.read().typecode(), 'u' | 'w') { + return Err(vm.new_value_error( + "fromunicode() may only be called on unicode type arrays ('u' or 'w')", + )); } let mut w = zelf.try_resizable(vm)?; let bytes = Self::_unicode_to_wchar_bytes(wtf8, w.itemsize()); @@ -838,10 +885,10 @@ pub mod array { #[pymethod] fn tounicode(&self, vm: &VirtualMachine) -> PyResult { let array = self.array.read(); - if array.typecode() != 'u' { - return Err( - vm.new_value_error("tounicode() may only be called on unicode type arrays") - ); + if !matches!(array.typecode(), 'u' | 'w') { + return Err(vm.new_value_error( + "tounicode() may only be called on unicode type arrays ('u' or 'w')", + )); } let bytes = array.get_bytes(); Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm) @@ -1152,7 +1199,7 @@ pub mod array { let array = zelf.read(); let cls = zelf.class().to_owned(); let typecode = vm.ctx.new_str(array.typecode_str()); - let values = if array.typecode() == 'u' { + let values = if matches!(array.typecode(), 'u' | 'w') { let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?; s.code_points().map(|x| x.to_pyobject(vm)).collect() } else { @@ -1266,13 +1313,14 @@ pub mod array { fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { let class = zelf.class(); let class_name = class.name(); - if zelf.read().typecode() == 'u' { + let typecode = zelf.read().typecode(); + if matches!(typecode, 'u' | 'w') { if zelf.__len__() == 0 { - return Ok(format!("{class_name}('u')")); + return Ok(format!("{class_name}('{typecode}')")); } let to_unicode = zelf.tounicode(vm)?; let escape = crate::vm::literal::escape::UnicodeEscape::new_repr(&to_unicode); - return Ok(format!("{}('u', {})", class_name, escape.str_repr())); + return Ok(format!("{class_name}('{typecode}', {})", escape.str_repr())); } zelf.read().repr(&class_name, vm) } @@ -1524,6 +1572,7 @@ pub mod array { _ => None, }; } + 'w' => return Some(Self::Utf32 { big_endian }), 'f' => { // Copied from CPython const Y: f32 = 16711938.0; diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index a174e2fd6..84c0ac17f 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -85,6 +85,7 @@ pub(crate) enum FormatType { UByte = b'B', Char = b'c', WideChar = b'u', + Ucs4Char = b'w', Str = b's', Pascal = b'p', Short = b'h', @@ -189,6 +190,7 @@ impl FormatType { unpack: Some(unpack_char), }, Self::WideChar => native_info!(wchar_t), + Self::Ucs4Char => native_info!(u32), Self::Short => native_info!(raw::c_short), Self::UShort => native_info!(raw::c_ushort), Self::Int => native_info!(raw::c_int), @@ -344,9 +346,10 @@ impl FormatCode { let code = FormatType::try_from(c) .ok() .filter(|c| match c { - FormatType::SSizeT | FormatType::SizeT | FormatType::VoidP => { - endianness == Endianness::Native - } + FormatType::SSizeT + | FormatType::SizeT + | FormatType::VoidP + | FormatType::Ucs4Char => endianness == Endianness::Native, _ => true, }) .ok_or_else(|| "bad char in struct format".to_owned())?;