Replace malformed unicode error helper calls with the _real variants (#8359)

* Replace malformed unicode error helper calls with the _real variants

new_unicode_decode_error / new_unicode_encode_error build the exception
from a bare message without running the initializer, so the result has
none of the five attributes a unicode error must carry and str() renders
as an empty string. Convert the non-Windows call sites in csv, socket,
getlogin and the fs-path decoders to the _real constructors, passing the
source bytes/str and the failing offset from the captured Utf8Error.

The Windows-gated sites (nt, mbcs/oem codecs) and the sites whose source
object is not reachable (uname, array) are left for follow-ups.

Assisted-by: Claude Code:claude-fable-5

* Report the full invalid UTF-8 span in the converted decode errors

The conversions used valid_up_to() + 1 for the decode-error end offset,
which under-reports multi-byte invalid sequences. Take the span from the
Utf8Error instead: valid_up_to() + error_len(), or the input length for a
truncated sequence (error_len() == None), matching CPython.

Assisted-by: Claude Code:claude-opus-4-8

crates/stdlib/src/csv.rs

@@ -60,6 +60,21 @@ mod _csv {
         vm.new_exception_msg(super::_csv::error(vm), msg.into())
     }
 
+    fn new_not_utf8_error(
+        vm: &VirtualMachine,
+        bytes: &[u8],
+        err: core::str::Utf8Error,
+    ) -> PyBaseExceptionRef {
+        vm.new_unicode_decode_error_real(
+            vm.ctx.new_str("utf-8"),
+            vm.ctx.new_bytes(bytes.to_vec()),
+            err.valid_up_to(),
+            err.error_len()
+                .map_or(bytes.len(), |n| err.valid_up_to() + n),
+            vm.ctx.new_str("csv not utf8"),
+        )
+    }
+
     #[pyattr]
     #[pyclass(module = "csv", name = "Dialect")]
     #[derive(Debug, PyPayload, Clone, Copy)]
@@ -1111,8 +1126,8 @@ mod _csv {
                 {
                     return Ok(vm.ctx.none());
                 }
-                let field = core::str::from_utf8(&field)
-                    .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
+                let field =
+                    core::str::from_utf8(&field).map_err(|e| new_not_utf8_error(vm, &field, e))?;
                 Ok(vm.ctx.new_str(field).into())
             })
             .collect()
@@ -1248,7 +1263,7 @@ mod _csv {
                     prev_end = end;
                     let s = core::str::from_utf8(&buffer[range.clone()])
                         // not sure if this is possible - the input was all strings
-                        .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?;
+                        .map_err(|e| new_not_utf8_error(vm, &buffer[range.clone()], e))?;
 
                     // TODO: RUSTPYTHON; Incomplete implementation
                     if let QuoteStyle::Nonnumeric = zelf.dialect.quoting {
@@ -1423,8 +1438,8 @@ mod _csv {
             }
 
             write_lineterminator(&mut output, self.dialect.lineterminator);
-            let s = core::str::from_utf8(&output)
-                .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
+            let s =
+                core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
             self.write.call((s,), vm)
         }
 
@@ -1469,8 +1484,8 @@ mod _csv {
 
             write_lineterminator(&mut output, self.dialect.lineterminator);
 
-            let s = core::str::from_utf8(&output)
-                .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
+            let s =
+                core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
 
             self.write.call((s,), vm)
         }
@@ -1518,8 +1533,8 @@ mod _csv {
 
             write_lineterminator(&mut output, self.dialect.lineterminator);
 
-            let s = core::str::from_utf8(&output)
-                .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
+            let s =
+                core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
 
             self.write.call((s,), vm)
         }
@@ -1593,7 +1608,7 @@ mod _csv {
             }
 
             let s = core::str::from_utf8(&buffer[..buffer_offset])
-                .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
+                .map_err(|e| new_not_utf8_error(vm, &buffer[..buffer_offset], e))?;
 
             self.write.call((s,), vm)
         }

crates/stdlib/src/socket.rs

@@ -2612,8 +2612,15 @@ mod _socket {
             }
             Some(ArgStrOrBytesLike::Buf(b)) => {
                 let bytes = b.borrow_buf();
-                let host_str = core::str::from_utf8(&bytes)
-                    .map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?;
+                let host_str = core::str::from_utf8(&bytes).map_err(|e| {
+                    vm.new_unicode_decode_error_real(
+                        vm.ctx.new_str("utf-8"),
+                        vm.ctx.new_bytes(bytes.to_vec()),
+                        e.valid_up_to(),
+                        e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
+                        vm.ctx.new_str("host bytes is not utf8"),
+                    )
+                })?;
                 Some(host_str.to_owned())
             }
             None => None,
@@ -2627,14 +2634,35 @@ mod _socket {
                     ArgStrOrBytesLike::Str(s) => {
                         // For str, check for surrogates and raise UnicodeEncodeError if found
                         s.to_str()
-                            .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))?
+                            .ok_or_else(|| {
+                                let start = s
+                                    .as_wtf8()
+                                    .code_points()
+                                    .position(|c| c.to_char().is_none())
+                                    .unwrap();
+                                vm.new_unicode_encode_error_real(
+                                    vm.ctx.new_str("utf-8"),
+                                    (*s).clone(),
+                                    start,
+                                    start + 1,
+                                    vm.ctx.new_str("surrogates not allowed"),
+                                )
+                            })?
                             .to_owned()
                     }
                     ArgStrOrBytesLike::Buf(b) => {
                         // For bytes, check if it's valid UTF-8
                         let bytes = b.borrow_buf();
                         core::str::from_utf8(&bytes)
-                            .map_err(|_| vm.new_unicode_decode_error("port is not utf8"))?
+                            .map_err(|e| {
+                                vm.new_unicode_decode_error_real(
+                                    vm.ctx.new_str("utf-8"),
+                                    vm.ctx.new_bytes(bytes.to_vec()),
+                                    e.valid_up_to(),
+                                    e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
+                                    vm.ctx.new_str("port is not utf8"),
+                                )
+                            })?
                             .to_owned()
                     }
                 };

crates/vm/src/function/fspath.rs

@@ -125,8 +125,15 @@ impl FsPath {
     }
 
     pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
-        rustpython_host_env::os::bytes_as_os_str(b)
-            .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
+        rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
+            vm.new_unicode_decode_error_real(
+                vm.ctx.new_str("utf-8"),
+                vm.ctx.new_bytes(b.to_vec()),
+                e.valid_up_to(),
+                e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
+                vm.ctx.new_str("can't decode path for utf-8"),
+            )
+        })
     }
 }
 

crates/vm/src/stdlib/os.rs

@@ -130,8 +130,15 @@ pub(super) struct FollowSymlinks(
 
 #[cfg(not(windows))]
 fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
-    rustpython_host_env::os::bytes_as_os_str(b)
-        .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
+    rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
+        vm.new_unicode_decode_error_real(
+            vm.ctx.new_str("utf-8"),
+            vm.ctx.new_bytes(b.to_vec()),
+            e.valid_up_to(),
+            e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
+            vm.ctx.new_str("can't decode path for utf-8"),
+        )
+    })
 }
 
 pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {

crates/vm/src/stdlib/posix.rs

@@ -1731,10 +1731,16 @@ pub mod module {
         let Some(login) = rustpython_host_env::posix::getlogin() else {
             return Err(vm.new_os_error("unable to determine login name"));
         };
-        login
-            .to_str()
-            .map(|s| s.to_owned())
-            .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}")))
+        login.to_str().map(|s| s.to_owned()).map_err(|e| {
+            vm.new_unicode_decode_error_real(
+                vm.ctx.new_str("utf-8"),
+                vm.ctx.new_bytes(login.as_bytes().to_vec()),
+                e.valid_up_to(),
+                e.error_len()
+                    .map_or(login.as_bytes().len(), |n| e.valid_up_to() + n),
+                vm.ctx.new_str("unable to decode login name"),
+            )
+        })
     }
 
     // cfg from nix