Fix malformed Unicode error constructors (#8497)

* Fix malformed Unicode error constructors

Replace message-only Unicode decode and encode errors with fully initialized exceptions, preserve source objects and failure ranges, and remove the obsolete constructors.

Assisted-by: Codex:gpt-5

* Fix Windows Unicode CI regressions

Assisted-by: Codex:gpt-5

Lib/test/test_codeccallbacks.py

@@ -1067,8 +1067,7 @@ class CodecCallbackTest(unittest.TestCase):
                 decoded = input.decode(enc, "test.bug36819")
                 self.assertEqual(decoded, 'abcdx' * 51)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON")
     def test_encodehelper_bug36819(self):
         handler = RepeatedPosReturn()
         codecs.register_error("test.bug36819", handler.handle)

crates/capi/src/pyerrors.rs

@@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create(
             unsafe { slice::from_raw_parts(object.cast::<u8>(), length) }.to_vec()
         };
 
-        let exc = vm.new_unicode_decode_error_real(
+        let exc = vm.new_unicode_decode_error(
             vm.ctx.new_str(encoding),
             vm.ctx.new_bytes(bytes),
             start,

crates/host_env/src/posix.rs

@@ -1,5 +1,4 @@
 use alloc::ffi::CString;
-#[cfg(all(unix, not(target_os = "redox")))]
 use alloc::vec::Vec;
 use core::ffi::CStr;
 #[cfg(all(unix, not(target_os = "redox")))]
@@ -22,6 +21,12 @@ pub struct UnameInfo {
     pub machine: String,
 }
 
+#[derive(Debug)]
+pub struct UnameDecodeError {
+    pub bytes: Vec<u8>,
+    pub error: core::str::Utf8Error,
+}
+
 #[cfg(all(unix, not(target_os = "redox")))]
 #[derive(Clone, Copy, Debug)]
 pub struct StatVfsInfo {
@@ -354,14 +359,23 @@ pub fn fchownat(
     .map_err(std::io::Error::from)
 }
 
-pub fn uname_info() -> Result<UnameInfo, core::str::Utf8Error> {
+pub fn uname_info() -> Result<UnameInfo, UnameDecodeError> {
+    fn decode(value: &CStr) -> Result<String, UnameDecodeError> {
+        core::str::from_utf8(value.to_bytes())
+            .map(str::to_owned)
+            .map_err(|error| UnameDecodeError {
+                bytes: value.to_bytes().to_vec(),
+                error,
+            })
+    }
+
     let info = rustix::system::uname();
     Ok(UnameInfo {
-        sysname: info.sysname().to_str()?.into(),
-        nodename: info.nodename().to_str()?.into(),
-        release: info.release().to_str()?.into(),
-        version: info.version().to_str()?.into(),
-        machine: info.machine().to_str()?.into(),
+        sysname: decode(info.sysname())?,
+        nodename: decode(info.nodename())?,
+        release: decode(info.release())?,
+        version: decode(info.version())?,
+        machine: decode(info.machine())?,
     })
 }
 

crates/stdlib/src/array.rs

@@ -657,7 +657,7 @@ pub mod array {
     impl ToPyResult for WideChar {
         fn to_pyresult(self, vm: &VirtualMachine) -> PyResult {
             Ok(CodePoint::try_from(self)
-                .map_err(|e| vm.new_unicode_encode_error(e))?
+                .map_err(|e| vm.new_value_error(e))?
                 .to_pyobject(vm))
         }
     }
@@ -1755,8 +1755,17 @@ pub mod array {
             })?,
             MachineFormatCode::Utf16 { big_endian } => {
                 let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect();
-                let s = String::from_utf16(&utf16)
-                    .map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?;
+                let s = String::from_utf16(&utf16).map_err(|_| {
+                    let (index, reason) = invalid_utf16(&utf16).unwrap();
+                    vm.new_unicode_decode_error(
+                        vm.ctx
+                            .new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }),
+                        args.items.clone(),
+                        index * 2,
+                        index * 2 + 2,
+                        vm.ctx.new_str(reason),
+                    )
+                })?;
                 let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize());
                 array.frombytes_move(bytes);
             }
@@ -1772,6 +1781,25 @@ pub mod array {
         PyArray::from(array).into_ref_with_type(vm, cls)
     }
 
+    fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> {
+        let mut index = 0;
+        while index < units.len() {
+            let unit = units[index];
+            if (0xd800..=0xdbff).contains(&unit) {
+                match units.get(index + 1) {
+                    Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2,
+                    Some(_) => return Some((index, "illegal UTF-16 surrogate")),
+                    None => return Some((index, "unexpected end of data")),
+                }
+            } else if (0xdc00..=0xdfff).contains(&unit) {
+                return Some((index, "illegal encoding"));
+            } else {
+                index += 1;
+            }
+        }
+        None
+    }
+
     // Register array.array as collections.abc.MutableSequence
     pub(crate) fn module_exec(
         vm: &VirtualMachine,

crates/stdlib/src/csv.rs

@@ -64,7 +64,7 @@ mod _csv {
         bytes: &[u8],
         err: core::str::Utf8Error,
     ) -> PyBaseExceptionRef {
-        vm.new_unicode_decode_error_real(
+        vm.new_unicode_decode_error(
             vm.ctx.new_str("utf-8"),
             vm.ctx.new_bytes(bytes.to_vec()),
             err.valid_up_to(),

crates/stdlib/src/socket.rs

@@ -2624,7 +2624,7 @@ mod _socket {
             Some(ArgStrOrBytesLike::Buf(b)) => {
                 let bytes = b.borrow_buf();
                 let host_str = core::str::from_utf8(&bytes).map_err(|e| {
-                    vm.new_unicode_decode_error_real(
+                    vm.new_unicode_decode_error(
                         vm.ctx.new_str("utf-8"),
                         vm.ctx.new_bytes(bytes.to_vec()),
                         e.valid_up_to(),
@@ -2666,7 +2666,7 @@ mod _socket {
                         let bytes = b.borrow_buf();
                         core::str::from_utf8(&bytes)
                             .map_err(|e| {
-                                vm.new_unicode_decode_error_real(
+                                vm.new_unicode_decode_error(
                                     vm.ctx.new_str("utf-8"),
                                     vm.ctx.new_bytes(bytes.to_vec()),
                                     e.valid_up_to(),

crates/stdlib/src/tkinter.rs

@@ -162,9 +162,18 @@ mod _tkinter {
 
         if let Some(tcl_obj) = obj.downcast_ref::<TclObject>() {
             let c_str = unsafe { tk_sys::Tcl_GetString(tcl_obj.value) };
-            let varname = unsafe { ffi::CStr::from_ptr(c_str as _) }
-                .to_str()
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?
+            let bytes = unsafe { ffi::CStr::from_ptr(c_str as _) }.to_bytes();
+            let varname = core::str::from_utf8(bytes)
+                .map_err(|e| {
+                    vm.new_unicode_decode_error(
+                        vm.ctx.new_str("utf-8"),
+                        vm.ctx.new_bytes(bytes.to_vec()),
+                        e.valid_up_to(),
+                        e.error_len()
+                            .map_or(bytes.len(), |len| e.valid_up_to() + len),
+                        vm.ctx.new_str(e.to_string()),
+                    )
+                })?
                 .to_owned();
             return Ok(varname);
         }

crates/vm/src/codecs.rs

@@ -802,7 +802,7 @@ impl DecodeContext for PyDecodeContext<'_> {
                     } else {
                         vm.ctx.new_bytes(self.data.to_vec())
                     };
-                    vm.new_unicode_decode_error_real(
+                    vm.new_unicode_decode_error(
                         vm.ctx.new_str(self.encoding),
                         data,
                         byte_range.start,

crates/vm/src/function/fspath.rs

@@ -125,7 +125,7 @@ impl FsPath {
 
     pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
         rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
-            vm.new_unicode_decode_error_real(
+            vm.new_unicode_decode_error(
                 vm.ctx.new_str("utf-8"),
                 vm.ctx.new_bytes(b.to_vec()),
                 e.valid_up_to(),

crates/vm/src/stdlib/_codecs.rs

@@ -382,6 +382,23 @@ mod _codecs_windows {
     use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike};
     use rustpython_host_env::windows as host_windows;
 
+    fn string_from_utf16(
+        encoding: &str,
+        data: &[u8],
+        wide: &[u16],
+        vm: &VirtualMachine,
+    ) -> PyResult<String> {
+        String::from_utf16(wide).map_err(|err| {
+            vm.new_unicode_decode_error(
+                vm.ctx.new_str(encoding),
+                vm.ctx.new_bytes(data.to_vec()),
+                0,
+                data.len(),
+                vm.ctx.new_str(format!("{encoding}_decode failed: {err}")),
+            )
+        })
+    }
+
     #[derive(FromArgs)]
     struct MbcsEncodeArgs {
         #[pyarg(positional)]
@@ -399,9 +416,7 @@ mod _codecs_windows {
             Some(s) => s,
             None => {
                 // String contains surrogates - not encodable with mbcs
-                return Err(vm.new_unicode_encode_error(
-                    "'mbcs' codec can't encode character: surrogates not allowed",
-                ));
+                return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
             }
         };
         let char_len = args.s.char_len();
@@ -433,9 +448,7 @@ mod _codecs_windows {
         .map_err(|err| vm.new_os_error(format!("mbcs_encode failed: {err}")))?;
 
         if errors == "strict" && used_default_char {
-            return Err(vm.new_unicode_encode_error(
-                "'mbcs' codec can't encode characters: invalid character",
-            ));
+            return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
         }
 
         buffer.truncate(result);
@@ -484,8 +497,7 @@ mod _codecs_windows {
             )
             .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
             buffer.truncate(result);
-            let s = String::from_utf16(&buffer)
-                .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
+            let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
             return Ok((s, len));
         }
 
@@ -500,8 +512,7 @@ mod _codecs_windows {
         )
         .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
         buffer.truncate(result);
-        let s = String::from_utf16(&buffer)
-            .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
+        let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
 
         Ok((s, len))
     }
@@ -523,9 +534,7 @@ mod _codecs_windows {
             Some(s) => s,
             None => {
                 // String contains surrogates - not encodable with oem
-                return Err(vm.new_unicode_encode_error(
-                    "'oem' codec can't encode character: surrogates not allowed",
-                ));
+                return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
             }
         };
         let char_len = args.s.char_len();
@@ -557,9 +566,7 @@ mod _codecs_windows {
         .map_err(|err| vm.new_os_error(format!("oem_encode failed: {err}")))?;
 
         if errors == "strict" && used_default_char {
-            return Err(vm.new_unicode_encode_error(
-                "'oem' codec can't encode characters: invalid character",
-            ));
+            return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
         }
 
         buffer.truncate(result);
@@ -609,8 +616,7 @@ mod _codecs_windows {
             )
             .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
             buffer.truncate(result);
-            let s = String::from_utf16(&buffer)
-                .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
+            let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
             return Ok((s, len));
         }
 
@@ -625,8 +631,7 @@ mod _codecs_windows {
         )
         .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
         buffer.truncate(result);
-        let s = String::from_utf16(&buffer)
-            .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
+        let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
 
         Ok((s, len))
     }
@@ -1024,7 +1029,7 @@ mod _codecs_windows {
                 }
             }
             let object = vm.ctx.new_bytes(data.to_vec());
-            return Err(vm.new_unicode_decode_error_real(
+            return Err(vm.new_unicode_decode_error(
                 encoding_str,
                 object,
                 fail_pos,
@@ -1115,7 +1120,7 @@ mod _codecs_windows {
                     }
                     "strict" => {
                         let object = vm.ctx.new_bytes(data.to_vec());
-                        return Err(vm.new_unicode_decode_error_real(
+                        return Err(vm.new_unicode_decode_error(
                             encoding_str,
                             object,
                             pos,
@@ -1126,7 +1131,7 @@ mod _codecs_windows {
                     _ => {
                         // Custom error handler
                         let object = vm.ctx.new_bytes(data.to_vec());
-                        let exc = vm.new_unicode_decode_error_real(
+                        let exc = vm.new_unicode_decode_error(
                             encoding_str.clone(),
                             object,
                             pos,

crates/vm/src/stdlib/nt.rs

@@ -19,7 +19,7 @@ pub(crate) mod module {
     use libc::intptr_t;
     use rustpython_common::wtf8::Wtf8Buf;
     use rustpython_host_env::nt as host_nt;
-    use std::os::windows::ffi::OsStringExt;
+    use std::os::windows::ffi::{OsStrExt, OsStringExt};
     use std::os::windows::io::AsRawHandle;
 
     #[pyattr]
@@ -49,6 +49,26 @@ pub(crate) mod module {
     #[pyattr]
     const TMP_MAX: i32 = i32::MAX;
 
+    fn utf8_from_bytes<'a>(bytes: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a str> {
+        core::str::from_utf8(bytes).map_err(|err| {
+            let reason = match err.error_len() {
+                None => "unexpected end of data",
+                Some(_) => match bytes[err.valid_up_to()] {
+                    0xc2..=0xf4 => "invalid continuation byte",
+                    _ => "invalid start byte",
+                },
+            };
+            vm.new_unicode_decode_error(
+                vm.ctx.new_str("utf-8"),
+                vm.ctx.new_bytes(bytes.to_vec()),
+                err.valid_up_to(),
+                err.error_len()
+                    .map_or(bytes.len(), |len| err.valid_up_to() + len),
+                vm.ctx.new_str(reason),
+            )
+        })
+    }
+
     #[pyattr]
     use host_nt::{
         LOAD_LIBRARY_SEARCH_APPLICATION_DIR as _LOAD_LIBRARY_SEARCH_APPLICATION_DIR,
@@ -214,11 +234,8 @@ pub(crate) mod module {
     fn _findfirstfile(path: OsPath, vm: &VirtualMachine) -> PyResult<PyStrRef> {
         let filename = host_nt::find_first_file_name(path.as_ref())
             .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?;
-        let filename_str = filename
-            .to_str()
-            .ok_or_else(|| vm.new_unicode_decode_error("filename contains invalid UTF-8"))?;
-
-        Ok(vm.ctx.new_str(filename_str))
+        let filename_wide: Vec<_> = filename.encode_wide().collect();
+        Ok(vm.ctx.new_str(Wtf8Buf::from_wide(&filename_wide)))
     }
 
     #[derive(FromArgs)]
@@ -689,17 +706,7 @@ pub(crate) mod module {
             (wide, false)
         } else if let Some(b) = path.downcast_ref::<PyBytes>() {
             // On Windows, bytes must be valid UTF-8 - this raises UnicodeDecodeError if not
-            let s = core::str::from_utf8(b.as_bytes()).map_err(|e| {
-                vm.new_exception_msg(
-                    vm.ctx.exceptions.unicode_decode_error.to_owned(),
-                    format!(
-                        "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte",
-                        b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0),
-                        e.valid_up_to()
-                    )
-                    .into(),
-                )
-            })?;
+            let s = utf8_from_bytes(b.as_bytes(), vm)?;
             let wide: Vec<u16> = s.encode_utf16().collect();
             (wide, true)
         } else {
@@ -720,16 +727,13 @@ pub(crate) mod module {
         // Return as bytes if input was bytes, preserving the original content
         if is_bytes {
             // Convert UTF-16 back to UTF-8 for bytes output
-            let drv = String::from_utf16(&wide[..drv_size])
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?;
-            let root = String::from_utf16(&wide[drv_size..drv_size + root_size])
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?;
-            let tail = String::from_utf16(&wide[drv_size + root_size..])
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?;
+            let drv = Wtf8Buf::from_wide(&wide[..drv_size]).into_bytes();
+            let root = Wtf8Buf::from_wide(&wide[drv_size..drv_size + root_size]).into_bytes();
+            let tail = Wtf8Buf::from_wide(&wide[drv_size + root_size..]).into_bytes();
             Ok(vm.ctx.new_tuple(vec![
-                vm.ctx.new_bytes(drv.into_bytes()).into(),
-                vm.ctx.new_bytes(root.into_bytes()).into(),
-                vm.ctx.new_bytes(tail.into_bytes()).into(),
+                vm.ctx.new_bytes(drv).into(),
+                vm.ctx.new_bytes(root).into(),
+                vm.ctx.new_bytes(tail).into(),
             ]))
         } else {
             // For str output, use WTF-8 to handle surrogates
@@ -913,17 +917,7 @@ pub(crate) mod module {
             let wide: Vec<u16> = s.as_wtf8().encode_wide().collect();
             (wide, false)
         } else if let Some(b) = path.downcast_ref::<PyBytes>() {
-            let s = core::str::from_utf8(b.as_bytes()).map_err(|e| {
-                vm.new_exception_msg(
-                    vm.ctx.exceptions.unicode_decode_error.to_owned(),
-                    format!(
-                        "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte",
-                        b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0),
-                        e.valid_up_to()
-                    )
-                    .into(),
-                )
-            })?;
+            let s = utf8_from_bytes(b.as_bytes(), vm)?;
             let wide: Vec<u16> = s.encode_utf16().collect();
             (wide, true)
         } else {
@@ -936,9 +930,8 @@ pub(crate) mod module {
         let normalized = normpath_wide(&wide);
 
         if is_bytes {
-            let s = String::from_utf16(&normalized)
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?;
-            Ok(vm.ctx.new_bytes(s.into_bytes()).into())
+            let bytes = Wtf8Buf::from_wide(&normalized).into_bytes();
+            Ok(vm.ctx.new_bytes(bytes).into())
         } else {
             let s = Wtf8Buf::from_wide(&normalized);
             Ok(vm.ctx.new_str(s).into())

crates/vm/src/stdlib/os.rs

@@ -131,7 +131,7 @@ pub(super) struct FollowSymlinks(
 #[cfg(not(windows))]
 fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
     rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
-        vm.new_unicode_decode_error_real(
+        vm.new_unicode_decode_error(
             vm.ctx.new_str("utf-8"),
             vm.ctx.new_bytes(b.to_vec()),
             e.valid_up_to(),

crates/vm/src/stdlib/posix.rs

@@ -1340,8 +1340,20 @@ pub mod module {
 
     #[pyfunction]
     fn uname(vm: &VirtualMachine) -> PyResult<_os::UnameResultData> {
-        let info = rustpython_host_env::posix::uname_info()
-            .map_err(|err| vm.new_unicode_decode_error(err.to_string()))?;
+        let info = rustpython_host_env::posix::uname_info().map_err(|err| {
+            let start = err.error.valid_up_to();
+            let end = err
+                .error
+                .error_len()
+                .map_or(err.bytes.len(), |len| start + len);
+            vm.new_unicode_decode_error(
+                vm.ctx.new_str("utf-8"),
+                vm.ctx.new_bytes(err.bytes),
+                start,
+                end,
+                vm.ctx.new_str(err.error.to_string()),
+            )
+        })?;
         Ok(_os::UnameResultData {
             sysname: info.sysname,
             nodename: info.nodename,
@@ -1791,7 +1803,7 @@ pub mod module {
             return Err(vm.new_os_error("unable to determine login name"));
         };
         login.to_str().map(|s| s.to_owned()).map_err(|e| {
-            vm.new_unicode_decode_error_real(
+            vm.new_unicode_decode_error(
                 vm.ctx.new_str("utf-8"),
                 vm.ctx.new_bytes(login.as_bytes().to_vec()),
                 e.valid_up_to(),

crates/vm/src/vm/vm_new.rs

@@ -517,7 +517,7 @@ impl VirtualMachine {
         self.new_os_subtype_error(exc_type.to_owned(), Some(errno), msg)
     }
 
-    pub fn new_unicode_decode_error_real(
+    pub fn new_unicode_decode_error(
         &self,
         encoding: PyStrRef,
         object: PyBytesRef,
@@ -996,12 +996,6 @@ impl VirtualMachine {
     define_exception_fn!(fn new_type_error, type_error, TypeError);
     define_exception_fn!(fn new_system_error, system_error, SystemError);
 
-    // TODO: remove & replace with new_unicode_decode_error_real
-    define_exception_fn!(fn new_unicode_decode_error, unicode_decode_error, UnicodeDecodeError);
-
-    // TODO: remove & replace with new_unicode_encode_error_real
-    define_exception_fn!(fn new_unicode_encode_error, unicode_encode_error, UnicodeEncodeError);
-
     define_exception_fn!(fn new_value_error, value_error, ValueError);
 
     define_exception_fn!(fn new_buffer_error, buffer_error, BufferError);