Add 'w' (Py_UCS4) typecode support to array (#8266)

* Support array 'w' (Py_UCS4) typecode

Python 3.13 added the 'w' typecode holding fixed 4-byte unicode code
points, unlike 'u' which is platform-dependent wchar_t. Add a Ucs4Char
element type and route it through the existing unicode-array paths:
constructor from str, fromunicode()/tounicode(), __reduce_ex__, repr,
byteswap, and the machine-format code (Utf32). Also register 'w' as a
struct format char in vm::buffer.

Remove the now-passing expectedFailure markers for 'w' tests in
test_array.py.

Assisted-by: Claude

* Reject non-native byte order for the 'w' format char

FormatType::parse accepted 'w' after '<', '>', '!', or '=', but the
non-native branch of FormatType::info has no Ucs4Char arm and hits
unreachable!(), so struct.calcsize('>w') panicked the interpreter.
Add Ucs4Char to the existing native-only filter in FormatCode::parse
(alongside SSizeT/SizeT/VoidP) so those formats fail with the regular
"bad char in struct format" error instead. Native 'w' and
memoryview(array('w', ...)) are unaffected.

Assisted-by: Claude

* Unmark 'w'-typecode tests in test_re and test_csv

test_re's test_empty_array and test_csv's test_char_write construct
array('w', ...) and were marked expectedFailure while the typecode was
unsupported. Now that 'w' works they pass, and regrtest counts the
unexpected successes as failures in CI. Remove the two markers, which
restores both tests to their CPython v3.14.6 form; test_re, test_csv,
and test_array all pass.

Assisted-by: Claude

Lib/test/test_array.py

@@ -189,7 +189,6 @@ class ArrayReconstructorTest(unittest.TestCase):
                 self.assertEqual(a, b,
                     msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase))
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
     def test_unicode(self):
         teststr = "Bonne Journ\xe9e \U0002030a\U00020347"
         testcases = (
@@ -1180,7 +1179,6 @@ class BaseTest:
         basesize = support.calcvobjsize('Pn2Pi')
         support.check_sizeof(self, a, basesize)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
     def test_initialize_with_unicode(self):
         if self.typecode not in ('u', 'w'):
             with self.assertRaises(TypeError) as cm:
@@ -1267,207 +1265,11 @@ class UnicodeTest(StringTest, unittest.TestCase):
                 self.assertEqual(len(a), 0)
                 self.assertEqual(a.typecode, 'u')
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_add(self):
-        return super().test_add()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_extend(self):
-        return super().test_extend()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_iadd(self):
-        return super().test_iadd()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_setiadd(self):
-        return super().test_setiadd()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_setslice(self):
-        return super().test_setslice()
-
 
 class UCS4Test(UnicodeTest):
     typecode = 'w'
     minitemsize = 4
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_buffer(self):
-        return super().test_buffer()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_buffer_info(self):
-        return super().test_buffer_info()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_byteswap(self):
-        return super().test_byteswap()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_clear(self):
-        return super().test_clear()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_cmp(self):
-        return super().test_cmp()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_constructor(self):
-        return super().test_constructor()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_constructor_with_iterable_argument(self):
-        return super().test_constructor_with_iterable_argument()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_copy(self):
-        return super().test_copy()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_count(self):
-        return super().test_count()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_coveritertraverse(self):
-        return super().test_coveritertraverse()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_deepcopy(self):
-        return super().test_deepcopy()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_delitem(self):
-        return super().test_delitem()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_exhausted_iterator(self):
-        return super().test_exhausted_iterator()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_exhausted_reverse_iterator(self):
-        return super().test_exhausted_reverse_iterator()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_extended_getslice(self):
-        return super().test_extended_getslice()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_extended_set_del_slice(self):
-        return super().test_extended_set_del_slice()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_filewrite(self):
-        return super().test_filewrite()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_fromarray(self):
-        return super().test_fromarray()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_fromfile_ioerror(self):
-        return super().test_fromfile_ioerror()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_getitem(self):
-        return super().test_getitem()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_getslice(self):
-        return super().test_getslice()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_imul(self):
-        return super().test_imul()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_index(self):
-        return super().test_index()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_insert(self):
-        return super().test_insert()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_issue17223(self):
-        return super().test_issue17223()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_iterator_pickle(self):
-        return super().test_iterator_pickle()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_len(self):
-        return super().test_len()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_mul(self):
-        return super().test_mul()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_pickle(self):
-        return super().test_pickle()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_pickle_for_empty_array(self):
-        return super().test_pickle_for_empty_array()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_pop(self):
-        return super().test_pop()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_reduce_ex(self):
-        return super().test_reduce_ex()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_remove(self):
-        return super().test_remove()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_repr(self):
-        return super().test_repr()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_reverse(self):
-        return super().test_reverse()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_reverse_iterator(self):
-        return super().test_reverse_iterator()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_reverse_iterator_picking(self):
-        return super().test_reverse_iterator_picking()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_setitem(self):
-        return super().test_setitem()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_str(self):
-        return super().test_str()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_tofrombytes(self):
-        return super().test_tofrombytes()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_tofromfile(self):
-        return super().test_tofromfile()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_tofromlist(self):
-        return super().test_tofromlist()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_unicode(self):
-        return super().test_unicode()
-
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Add support for 'w'
-    def test_weakref(self):
-        return super().test_weakref()
-
 
 class NumberTest(BaseTest):
 

Lib/test/test_csv.py

@@ -1137,7 +1137,6 @@ class TestArrayWrites(unittest.TestCase):
             fileobj.seek(0)
             self.assertEqual(fileobj.read(), expected)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_char_write(self):
         import array, string
         a = array.array('w', string.ascii_letters)

Lib/test/test_re.py

@@ -1754,7 +1754,6 @@ class ReTests(unittest.TestCase):
         for x in not_decimal_digits:
             self.assertIsNone(re.match(r'^\d$', x))
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; a = array.array(typecode)\n ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)
     @warnings_helper.ignore_warnings(category=DeprecationWarning)  # gh-80480 array('u')
     def test_empty_array(self):
         # SF buf 1647541

crates/stdlib/src/array.rs

@@ -60,7 +60,7 @@ pub mod array {
                     match c {
                         $($c => Ok(ArrayContentType::$n(Vec::new())),)*
                         _ => Err(
-                            "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)".into()
+                            "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)".into()
                         ),
                     }
                 }
@@ -473,6 +473,7 @@ pub mod array {
         (SignedByte, i8, 'b', "b"),
         (UnsignedByte, u8, 'B', "B"),
         (PyUnicode, WideChar, 'u', "u"),
+        (PyUcs4, Ucs4Char, 'w', "w"),
         (SignedShort, raw::c_short, 'h', "h"),
         (UnsignedShort, raw::c_ushort, 'H', "H"),
         (SignedInt, raw::c_int, 'i', "i"),
@@ -488,6 +489,11 @@ pub mod array {
     #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)]
     pub struct WideChar(wchar_t);
 
+    /// Element type for the 'w' typecode: always a 4-byte unicode code point
+    /// (Py_UCS4), unlike 'u' which is platform-dependent `wchar_t`.
+    #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)]
+    pub struct Ucs4Char(u32);
+
     trait ArrayElement: Sized {
         fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self>;
         fn byteswap(self) -> Self;
@@ -574,6 +580,47 @@ pub mod array {
         }
     }
 
+    impl ArrayElement for Ucs4Char {
+        fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+            let s = obj.downcast::<PyStr>().map_err(|obj| {
+                vm.new_type_error(format!(
+                    "array item must be a unicode character, not {}",
+                    obj.class().name()
+                ))
+            })?;
+            s.as_wtf8()
+                .code_points()
+                .exactly_one()
+                .map(|ch| Self(ch.to_u32()))
+                .map_err(|e| {
+                    vm.new_type_error(format!(
+                        "array item must be a unicode character, not a string of length {}",
+                        e.count()
+                    ))
+                })
+        }
+        fn byteswap(self) -> Self {
+            Self(self.0.swap_bytes())
+        }
+        fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef {
+            unreachable!()
+        }
+    }
+
+    impl ToPyResult for Ucs4Char {
+        fn to_pyresult(self, vm: &VirtualMachine) -> PyResult {
+            Ok(u32_to_char(self.0)
+                .map_err(|msg| vm.new_value_error(msg))?
+                .to_pyobject(vm))
+        }
+    }
+
+    impl fmt::Display for Ucs4Char {
+        fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            unreachable!("`repr(array('w'))` calls `PyStr::repr`")
+        }
+    }
+
     fn u32_to_char(ch: u32) -> Result<CodePoint, String> {
         CodePoint::from_u32(ch)
             .ok_or_else(|| format!("character U+{ch:4x} is not in range [U+0000; U+10ffff]"))
@@ -663,7 +710,7 @@ pub mod array {
                     let value = init.read().typecode();
                     match (spec, value) {
                         (spec, ch) if spec == ch => array.frombytes(&init.get_bytes()),
-                        (spec, 'u') => {
+                        (spec, 'u' | 'w') if !matches!(spec, 'u' | 'w') => {
                             return Err(vm.new_type_error(format!(
                             "cannot use a unicode array to initialize an array with typecode '{spec}'"
                         )))
@@ -675,7 +722,7 @@ pub mod array {
                         }
                     }
                 } else if let Some(wtf8) = init.downcast_ref::<PyStr>() {
-                    if spec == 'u' {
+                    if matches!(spec, 'u' | 'w') {
                         let bytes = Self::_unicode_to_wchar_bytes(wtf8.as_wtf8(), array.itemsize());
                         array.frombytes_move(bytes);
                     } else {
@@ -824,10 +871,10 @@ pub mod array {
                     obj.class().name()
                 ))
             })?;
-            if zelf.read().typecode() != 'u' {
-                return Err(
-                    vm.new_value_error("fromunicode() may only be called on unicode type arrays")
-                );
+            if !matches!(zelf.read().typecode(), 'u' | 'w') {
+                return Err(vm.new_value_error(
+                    "fromunicode() may only be called on unicode type arrays ('u' or 'w')",
+                ));
             }
             let mut w = zelf.try_resizable(vm)?;
             let bytes = Self::_unicode_to_wchar_bytes(wtf8, w.itemsize());
@@ -838,10 +885,10 @@ pub mod array {
         #[pymethod]
         fn tounicode(&self, vm: &VirtualMachine) -> PyResult<Wtf8Buf> {
             let array = self.array.read();
-            if array.typecode() != 'u' {
-                return Err(
-                    vm.new_value_error("tounicode() may only be called on unicode type arrays")
-                );
+            if !matches!(array.typecode(), 'u' | 'w') {
+                return Err(vm.new_value_error(
+                    "tounicode() may only be called on unicode type arrays ('u' or 'w')",
+                ));
             }
             let bytes = array.get_bytes();
             Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm)
@@ -1152,7 +1199,7 @@ pub mod array {
             let array = zelf.read();
             let cls = zelf.class().to_owned();
             let typecode = vm.ctx.new_str(array.typecode_str());
-            let values = if array.typecode() == 'u' {
+            let values = if matches!(array.typecode(), 'u' | 'w') {
                 let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?;
                 s.code_points().map(|x| x.to_pyobject(vm)).collect()
             } else {
@@ -1266,13 +1313,14 @@ pub mod array {
         fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
             let class = zelf.class();
             let class_name = class.name();
-            if zelf.read().typecode() == 'u' {
+            let typecode = zelf.read().typecode();
+            if matches!(typecode, 'u' | 'w') {
                 if zelf.__len__() == 0 {
-                    return Ok(format!("{class_name}('u')"));
+                    return Ok(format!("{class_name}('{typecode}')"));
                 }
                 let to_unicode = zelf.tounicode(vm)?;
                 let escape = crate::vm::literal::escape::UnicodeEscape::new_repr(&to_unicode);
-                return Ok(format!("{}('u', {})", class_name, escape.str_repr()));
+                return Ok(format!("{class_name}('{typecode}', {})", escape.str_repr()));
             }
             zelf.read().repr(&class_name, vm)
         }
@@ -1524,6 +1572,7 @@ pub mod array {
                         _ => None,
                     };
                 }
+                'w' => return Some(Self::Utf32 { big_endian }),
                 'f' => {
                     // Copied from CPython
                     const Y: f32 = 16711938.0;

crates/vm/src/buffer.rs

@@ -85,6 +85,7 @@ pub(crate) enum FormatType {
     UByte = b'B',
     Char = b'c',
     WideChar = b'u',
+    Ucs4Char = b'w',
     Str = b's',
     Pascal = b'p',
     Short = b'h',
@@ -189,6 +190,7 @@ impl FormatType {
                     unpack: Some(unpack_char),
                 },
                 Self::WideChar => native_info!(wchar_t),
+                Self::Ucs4Char => native_info!(u32),
                 Self::Short => native_info!(raw::c_short),
                 Self::UShort => native_info!(raw::c_ushort),
                 Self::Int => native_info!(raw::c_int),
@@ -344,9 +346,10 @@ impl FormatCode {
             let code = FormatType::try_from(c)
                 .ok()
                 .filter(|c| match c {
-                    FormatType::SSizeT | FormatType::SizeT | FormatType::VoidP => {
-                        endianness == Endianness::Native
-                    }
+                    FormatType::SSizeT
+                    | FormatType::SizeT
+                    | FormatType::VoidP
+                    | FormatType::Ucs4Char => endianness == Endianness::Native,
                     _ => true,
                 })
                 .ok_or_else(|| "bad char in struct format".to_owned())?;