Merge pull request #5587 from coolreader18/wtf8

Allow surrogates in str

Cargo.lock

@@ -2338,6 +2338,7 @@ version = "0.4.0"
 dependencies = [
  "ascii",
  "bitflags 2.8.0",
+ "bstr",
  "cfg-if",
  "itertools 0.14.0",
  "libc",
@@ -2345,6 +2346,7 @@ dependencies = [
  "malachite-base",
  "malachite-bigint",
  "malachite-q",
+ "memchr",
  "num-complex",
  "num-traits",
  "once_cell",
@@ -2472,6 +2474,7 @@ dependencies = [
  "criterion",
  "num_enum",
  "optional",
+ "rustpython-common",
 ]
 
 [[package]]

Lib/test/string_tests.py

@@ -1066,8 +1066,6 @@ class CommonTest(BaseTest):
             hash(b)
         self.assertEqual(hash(a), hash(b))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_capitalize_nonascii(self):
         # check that titlecased chars are lowered correctly
         # \u1ffc is the titlecased char

Lib/test/test_cmd_line_script.py

@@ -574,6 +574,7 @@ class CmdLineTest(unittest.TestCase):
             self.assertTrue(text[1].startswith('  File '))
             self.assertTrue(text[3].startswith('NameError'))
 
+    @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON")
     def test_non_ascii(self):
         # Mac OS X denies the creation of a file with an invalid UTF-8 name.
         # Windows allows creating a name with an arbitrary bytes name, but

Lib/test/test_codecs.py

@@ -1698,8 +1698,6 @@ nameprep_tests = [
 
 
 class NameprepTest(unittest.TestCase):
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_nameprep(self):
         from encodings.idna import nameprep
         for pos, (orig, prepped) in enumerate(nameprep_tests):

Lib/test/test_difflib.py

@@ -373,8 +373,6 @@ class TestBytes(unittest.TestCase):
         check(difflib.diff_bytes(context, a, a, b'a', b'a', b'2005', b'2013'))
         check(difflib.diff_bytes(context, a, b, b'a', b'b', b'2005', b'2013'))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_byte_filenames(self):
         # somebody renamed a file from ISO-8859-2 to UTF-8
         fna = b'\xb3odz.txt'    # "łodz.txt"

Lib/test/test_import/__init__.py

@@ -1305,6 +1305,8 @@ class ImportTracebackTests(unittest.TestCase):
             else:
                 importlib.SourceLoader.exec_module = old_exec_module
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE')
     def test_unencodable_filename(self):
         # Issue #11619: The Python parser and the import machinery must not

Lib/test/test_json/test_scanstring.py

@@ -143,10 +143,4 @@ class TestScanstring:
 
 
 class TestPyScanstring(TestScanstring, PyTest): pass
-# TODO: RUSTPYTHON
-class TestPyScanstring(TestScanstring, PyTest):
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    def test_bad_escapes(self):
-        super().test_bad_escapes()
 class TestCScanstring(TestScanstring, CTest): pass

Lib/test/test_ntpath.py

@@ -1032,12 +1032,6 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     pathmodule = ntpath
     attributes = ['relpath']
 
-    # TODO: RUSTPYTHON
-    if sys.platform == "linux":
-        @unittest.expectedFailure
-        def test_nonascii_abspath(self):
-            super().test_nonascii_abspath()
-
     # TODO: RUSTPYTHON
     if sys.platform == "win32":
         # TODO: RUSTPYTHON, ValueError: illegal environment variable name

Lib/test/test_re.py

@@ -854,8 +854,6 @@ class ReTests(unittest.TestCase):
         # Can match around the whitespace.
         self.assertEqual(len(re.findall(r"\B", " ")), 2)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_bigcharset(self):
         self.assertEqual(re.match("([\u2222\u2223])",
                                   "\u2222").group(1), "\u2222")
@@ -2233,6 +2231,7 @@ class ReTests(unittest.TestCase):
         with self.assertRaisesRegex(TypeError, "got 'type'"):
             re.search("x*", type)
 
+    @unittest.skip("TODO: RUSTPYTHON: flaky, improve perf")
     @requires_resource('cpu')
     def test_search_anchor_at_beginning(self):
         s = 'x'*10**7

Lib/test/test_smtplib.py

@@ -1459,8 +1459,6 @@ class SMTPUTF8SimTests(unittest.TestCase):
         self.assertIn('SMTPUTF8', self.serv.last_mail_options)
         self.assertEqual(self.serv.last_rcpt_options, [])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_send_message_uses_smtputf8_if_addrs_non_ascii(self):
         msg = EmailMessage()
         msg['From'] = "Páolo <főo@bar.com>"

Lib/test/test_socket.py

@@ -1578,7 +1578,7 @@ class GeneralModuleTests(unittest.TestCase):
         # only IP addresses are allowed
         self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0)
 
-    @unittest.expectedFailureIf(sys.platform != "darwin", "TODO: RUSTPYTHON; socket.gethostbyname_ex")
+    @unittest.skip("TODO: RUSTPYTHON: flaky on CI?")
     @unittest.skipUnless(support.is_resource_enabled('network'),
                          'network is not enabled')
     def test_idna(self):
@@ -5519,8 +5519,6 @@ class TestUnixDomain(unittest.TestCase):
         self.addCleanup(os_helper.unlink, path)
         self.assertEqual(self.sock.getsockname(), path)
 
-    # TODO: RUSTPYTHON, surrogateescape
-    @unittest.expectedFailure
     def testSurrogateescapeBind(self):
         # Test binding to a valid non-ASCII pathname, with the
         # non-ASCII bytes supplied using surrogateescape encoding.

Lib/test/test_sqlite3/test_types.py

@@ -95,8 +95,6 @@ class SqliteTypeTests(unittest.TestCase):
         row = self.cur.fetchone()
         self.assertIsNone(row)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_string_with_surrogates(self):
         for value in 0xd8ff, 0xdcff:
             with self.assertRaises(UnicodeEncodeError):

Lib/test/test_ucn.py

@@ -102,8 +102,6 @@ class UnicodeNamesTest(unittest.TestCase):
         self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D")
         self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_bmp_characters(self):
         for code in range(0x10000):
             char = chr(code)

Lib/test/test_unicode.py

@@ -721,8 +721,6 @@ class UnicodeTest(string_tests.CommonTest,
                    '\U0001F40D', '\U0001F46F']:
             self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @support.requires_resource('cpu')
     def test_isspace_invariant(self):
         for codepoint in range(sys.maxunicode + 1):

Lib/test/test_unicodedata.py

@@ -99,8 +99,6 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         result = h.hexdigest()
         self.assertEqual(result, self.expectedchecksum)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @requires_resource('cpu')
     def test_name_inverse_lookup(self):
         for i in range(sys.maxunicode + 1):
@@ -326,8 +324,6 @@ class UnicodeMiscTest(UnicodeDatabaseTest):
         self.assertTrue("\u1d79".upper()=='\ua77d')
         self.assertTrue(".".upper()=='.')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_bug_5828(self):
         self.assertEqual("\u1d79".lower(), "\u1d79")
         # Only U+0000 should have U+0000 as its upper/lower/titlecase variant
@@ -347,8 +343,6 @@ class UnicodeMiscTest(UnicodeDatabaseTest):
         self.assertEqual("\u01c5".title(), "\u01c5")
         self.assertEqual("\u01c6".title(), "\u01c5")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_linebreak_7643(self):
         for i in range(0x10000):
             lines = (chr(i) + 'A').splitlines()

common/Cargo.toml

@@ -16,12 +16,14 @@ rustpython-literal = { workspace = true }
 
 ascii = { workspace = true }
 bitflags = { workspace = true }
+bstr = { workspace = true }
 cfg-if = { workspace = true }
 itertools = { workspace = true }
 libc = { workspace = true }
 malachite-bigint = { workspace = true }
 malachite-q = { workspace = true }
 malachite-base = { workspace = true }
+memchr = { workspace = true }
 num-complex = { workspace = true }
 num-traits = { workspace = true }
 once_cell = { workspace = true }

common/src/cformat.rs

@@ -11,11 +11,13 @@ use std::{
     str::FromStr,
 };
 
+use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
+
 #[derive(Debug, PartialEq)]
 pub enum CFormatErrorType {
     UnmatchedKeyParentheses,
     MissingModuloSign,
-    UnsupportedFormatChar(char),
+    UnsupportedFormatChar(CodePoint),
     IncompleteFormat,
     IntTooBig,
     // Unimplemented,
@@ -39,7 +41,9 @@ impl fmt::Display for CFormatError {
             UnsupportedFormatChar(c) => write!(
                 f,
                 "unsupported format character '{}' ({:#x}) at index {}",
-                c, c as u32, self.index
+                c,
+                c.to_u32(),
+                self.index
             ),
             IntTooBig => write!(f, "width/precision too big"),
             _ => write!(f, "unexpected error parsing format string"),
@@ -160,7 +164,7 @@ pub trait FormatBuf:
     fn concat(self, other: Self) -> Self;
 }
 
-pub trait FormatChar: Copy + Into<char> + From<u8> {
+pub trait FormatChar: Copy + Into<CodePoint> + From<u8> {
     fn to_char_lossy(self) -> char;
     fn eq_char(self, c: char) -> bool;
 }
@@ -188,6 +192,29 @@ impl FormatChar for char {
     }
 }
 
+impl FormatBuf for Wtf8Buf {
+    type Char = CodePoint;
+    fn chars(&self) -> impl Iterator<Item = Self::Char> {
+        self.code_points()
+    }
+    fn len(&self) -> usize {
+        (**self).len()
+    }
+    fn concat(mut self, other: Self) -> Self {
+        self.extend([other]);
+        self
+    }
+}
+
+impl FormatChar for CodePoint {
+    fn to_char_lossy(self) -> char {
+        self.to_char_lossy()
+    }
+    fn eq_char(self, c: char) -> bool {
+        self == c
+    }
+}
+
 impl FormatBuf for Vec<u8> {
     type Char = u8;
     fn chars(&self) -> impl Iterator<Item = Self::Char> {
@@ -801,6 +828,15 @@ impl FromStr for CFormatString {
     }
 }
 
+pub type CFormatWtf8 = CFormatStrOrBytes<Wtf8Buf>;
+
+impl CFormatWtf8 {
+    pub fn parse_from_wtf8(s: &Wtf8) -> Result<Self, CFormatError> {
+        let mut iter = s.code_points().enumerate().peekable();
+        Self::parse(&mut iter)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

common/src/encodings.rs

@@ -1,12 +1,22 @@
 use std::ops::Range;
 
+use num_traits::ToPrimitive;
+
+use crate::str::StrKind;
+use crate::wtf8::{Wtf8, Wtf8Buf};
+
 pub type EncodeErrorResult<S, B, E> = Result<(EncodeReplace<S, B>, usize), E>;
 
 pub type DecodeErrorResult<S, B, E> = Result<(S, Option<B>, usize), E>;
 
-pub trait StrBuffer: AsRef<str> {
-    fn is_ascii(&self) -> bool {
-        self.as_ref().is_ascii()
+pub trait StrBuffer: AsRef<Wtf8> {
+    fn is_compatible_with(&self, kind: StrKind) -> bool {
+        let s = self.as_ref();
+        match kind {
+            StrKind::Ascii => s.is_ascii(),
+            StrKind::Utf8 => s.is_utf8(),
+            StrKind::Wtf8 => true,
+        }
     }
 }
 
@@ -16,7 +26,7 @@ pub trait ErrorHandler {
     type BytesBuf: AsRef<[u8]>;
     fn handle_encode_error(
         &self,
-        data: &str,
+        data: &Wtf8,
         char_range: Range<usize>,
         reason: &str,
     ) -> EncodeErrorResult<Self::StrBuf, Self::BytesBuf, Self::Error>;
@@ -27,7 +37,7 @@ pub trait ErrorHandler {
         reason: &str,
     ) -> DecodeErrorResult<Self::StrBuf, Self::BytesBuf, Self::Error>;
     fn error_oob_restart(&self, i: usize) -> Self::Error;
-    fn error_encoding(&self, data: &str, char_range: Range<usize>, reason: &str) -> Self::Error;
+    fn error_encoding(&self, data: &Wtf8, char_range: Range<usize>, reason: &str) -> Self::Error;
 }
 pub enum EncodeReplace<S, B> {
     Str(S),
@@ -63,19 +73,19 @@ fn decode_utf8_compatible<E: ErrorHandler, DecodeF, ErrF>(
     errors: &E,
     decode: DecodeF,
     handle_error: ErrF,
-) -> Result<(String, usize), E::Error>
+) -> Result<(Wtf8Buf, usize), E::Error>
 where
     DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>,
     ErrF: Fn(&[u8], Option<usize>) -> HandleResult<'_>,
 {
     if data.is_empty() {
-        return Ok((String::new(), 0));
+        return Ok((Wtf8Buf::new(), 0));
     }
     // we need to coerce the lifetime to that of the function body rather than the
     // anonymous input lifetime, so that we can assign it data borrowed from data_from_err
     let mut data = data;
     let mut data_from_err: E::BytesBuf;
-    let mut out = String::with_capacity(data.len());
+    let mut out = Wtf8Buf::with_capacity(data.len());
     let mut remaining_index = 0;
     let mut remaining_data = data;
     loop {
@@ -98,7 +108,7 @@ where
                             err_idx..err_len.map_or_else(|| data.len(), |len| err_idx + len);
                         let (replace, new_data, restart) =
                             errors.handle_decode_error(data, err_range, reason)?;
-                        out.push_str(replace.as_ref());
+                        out.push_wtf8(replace.as_ref());
                         if let Some(new_data) = new_data {
                             data_from_err = new_data;
                             data = data_from_err.as_ref();
@@ -116,21 +126,68 @@ where
     Ok((out, remaining_index))
 }
 
+#[inline]
+fn encode_utf8_compatible<E: ErrorHandler>(
+    s: &Wtf8,
+    errors: &E,
+    err_reason: &str,
+    target_kind: StrKind,
+) -> Result<Vec<u8>, E::Error> {
+    let full_data = s;
+    let mut data = s;
+    let mut char_data_index = 0;
+    let mut out = Vec::<u8>::new();
+    while let Some((char_i, (byte_i, _))) = data
+        .code_point_indices()
+        .enumerate()
+        .find(|(_, (_, c))| !target_kind.can_encode(*c))
+    {
+        out.extend_from_slice(&data.as_bytes()[..byte_i]);
+        let char_start = char_data_index + char_i;
+
+        // number of non-compatible chars between the first non-compatible char and the next compatible char
+        let non_compat_run_length = data[byte_i..]
+            .code_points()
+            .take_while(|c| !target_kind.can_encode(*c))
+            .count();
+        let char_range = char_start..char_start + non_compat_run_length;
+        let (replace, char_restart) =
+            errors.handle_encode_error(full_data, char_range.clone(), err_reason)?;
+        match replace {
+            EncodeReplace::Str(s) => {
+                if s.is_compatible_with(target_kind) {
+                    out.extend_from_slice(s.as_ref().as_bytes());
+                } else {
+                    return Err(errors.error_encoding(full_data, char_range, err_reason));
+                }
+            }
+            EncodeReplace::Bytes(b) => {
+                out.extend_from_slice(b.as_ref());
+            }
+        }
+        data = crate::str::try_get_codepoints(full_data, char_restart..)
+            .ok_or_else(|| errors.error_oob_restart(char_restart))?;
+        char_data_index = char_restart;
+    }
+    out.extend_from_slice(data.as_bytes());
+    Ok(out)
+}
+
 pub mod utf8 {
     use super::*;
 
     pub const ENCODING_NAME: &str = "utf-8";
 
     #[inline]
-    pub fn encode<E: ErrorHandler>(s: &str, _errors: &E) -> Result<Vec<u8>, E::Error> {
-        Ok(s.as_bytes().to_vec())
+    pub fn encode<E: ErrorHandler>(s: &Wtf8, errors: &E) -> Result<Vec<u8>, E::Error> {
+        encode_utf8_compatible(s, errors, "surrogates not allowed", StrKind::Utf8)
     }
 
     pub fn decode<E: ErrorHandler>(
         data: &[u8],
         errors: &E,
         final_decode: bool,
-    ) -> Result<(String, usize), E::Error> {
+    ) -> Result<(Wtf8Buf, usize), E::Error> {
         decode_utf8_compatible(
             data,
             errors,
@@ -180,14 +237,14 @@ pub mod latin_1 {
     const ERR_REASON: &str = "ordinal not in range(256)";
 
     #[inline]
-    pub fn encode<E: ErrorHandler>(s: &str, errors: &E) -> Result<Vec<u8>, E::Error> {
+    pub fn encode<E: ErrorHandler>(s: &Wtf8, errors: &E) -> Result<Vec<u8>, E::Error> {
         let full_data = s;
         let mut data = s;
         let mut char_data_index = 0;
         let mut out = Vec::<u8>::new();
         loop {
             match data
-                .char_indices()
+                .code_point_indices()
                 .enumerate()
                 .find(|(_, (_, c))| !c.is_ascii())
             {
@@ -198,17 +255,16 @@ pub mod latin_1 {
                 Some((char_i, (byte_i, ch))) => {
                     out.extend_from_slice(&data.as_bytes()[..byte_i]);
                     let char_start = char_data_index + char_i;
-                    if (ch as u32) <= 255 {
-                        out.push(ch as u8);
-                        let char_restart = char_start + 1;
-                        data = crate::str::try_get_chars(full_data, char_restart..)
-                            .ok_or_else(|| errors.error_oob_restart(char_restart))?;
-                        char_data_index = char_restart;
+                    if let Some(byte) = ch.to_u32().to_u8() {
+                        out.push(byte);
+                        // if the codepoint is between 128..=255, it's utf8-length is 2
+                        data = &data[byte_i + 2..];
+                        char_data_index = char_start + 1;
                     } else {
                         // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char
                         let non_latin_1_run_length = data[byte_i..]
-                            .chars()
-                            .take_while(|c| (*c as u32) > 255)
+                            .code_points()
+                            .take_while(|c| c.to_u32() > 255)
                             .count();
                         let char_range = char_start..char_start + non_latin_1_run_length;
                         let (replace, char_restart) = errors.handle_encode_error(
@@ -218,7 +274,7 @@ pub mod latin_1 {
                         )?;
                         match replace {
                             EncodeReplace::Str(s) => {
-                                if s.as_ref().chars().any(|c| (c as u32) > 255) {
+                                if s.as_ref().code_points().any(|c| c.to_u32() > 255) {
                                     return Err(
                                         errors.error_encoding(full_data, char_range, ERR_REASON)
                                     );
@@ -229,7 +285,7 @@ pub mod latin_1 {
                                 out.extend_from_slice(b.as_ref());
                             }
                         }
-                        data = crate::str::try_get_chars(full_data, char_restart..)
+                        data = crate::str::try_get_codepoints(full_data, char_restart..)
                             .ok_or_else(|| errors.error_oob_restart(char_restart))?;
                         char_data_index = char_restart;
                     }
@@ -240,10 +296,10 @@ pub mod latin_1 {
         Ok(out)
     }
 
-    pub fn decode<E: ErrorHandler>(data: &[u8], _errors: &E) -> Result<(String, usize), E::Error> {
+    pub fn decode<E: ErrorHandler>(data: &[u8], _errors: &E) -> Result<(Wtf8Buf, usize), E::Error> {
         let out: String = data.iter().map(|c| *c as char).collect();
         let out_len = out.len();
-        Ok((out, out_len))
+        Ok((out.into(), out_len))
     }
 }
 
@@ -256,54 +312,11 @@ pub mod ascii {
     const ERR_REASON: &str = "ordinal not in range(128)";
 
     #[inline]
-    pub fn encode<E: ErrorHandler>(s: &str, errors: &E) -> Result<Vec<u8>, E::Error> {
-        let full_data = s;
-        let mut data = s;
-        let mut char_data_index = 0;
-        let mut out = Vec::<u8>::new();
-        loop {
-            match data
-                .char_indices()
-                .enumerate()
-                .find(|(_, (_, c))| !c.is_ascii())
-            {
-                None => {
-                    out.extend_from_slice(data.as_bytes());
-                    break;
-                }
-                Some((char_i, (byte_i, _))) => {
-                    out.extend_from_slice(&data.as_bytes()[..byte_i]);
-                    let char_start = char_data_index + char_i;
-                    // number of non-ascii chars between the first non-ascii char and the next ascii char
-                    let non_ascii_run_length =
-                        data[byte_i..].chars().take_while(|c| !c.is_ascii()).count();
-                    let char_range = char_start..char_start + non_ascii_run_length;
-                    let (replace, char_restart) =
-                        errors.handle_encode_error(full_data, char_range.clone(), ERR_REASON)?;
-                    match replace {
-                        EncodeReplace::Str(s) => {
-                            if !s.is_ascii() {
-                                return Err(
-                                    errors.error_encoding(full_data, char_range, ERR_REASON)
-                                );
-                            }
-                            out.extend_from_slice(s.as_ref().as_bytes());
-                        }
-                        EncodeReplace::Bytes(b) => {
-                            out.extend_from_slice(b.as_ref());
-                        }
-                    }
-                    data = crate::str::try_get_chars(full_data, char_restart..)
-                        .ok_or_else(|| errors.error_oob_restart(char_restart))?;
-                    char_data_index = char_restart;
-                    continue;
-                }
-            }
-        }
-        Ok(out)
+    pub fn encode<E: ErrorHandler>(s: &Wtf8, errors: &E) -> Result<Vec<u8>, E::Error> {
+        encode_utf8_compatible(s, errors, ERR_REASON, StrKind::Ascii)
     }
 
-    pub fn decode<E: ErrorHandler>(data: &[u8], errors: &E) -> Result<(String, usize), E::Error> {
+    pub fn decode<E: ErrorHandler>(data: &[u8], errors: &E) -> Result<(Wtf8Buf, usize), E::Error> {
         decode_utf8_compatible(
             data,
             errors,

common/src/format.rs

@@ -7,8 +7,10 @@ use rustpython_literal::format::Case;
 use std::ops::Deref;
 use std::{cmp, str::FromStr};
 
+use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
+
 trait FormatParse {
-    fn parse(text: &str) -> (Option<Self>, &str)
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8)
     where
         Self: Sized;
 }
@@ -23,20 +25,20 @@ pub enum FormatConversion {
 }
 
 impl FormatParse for FormatConversion {
-    fn parse(text: &str) -> (Option<Self>, &str) {
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
         let Some(conversion) = Self::from_string(text) else {
             return (None, text);
         };
-        let mut chars = text.chars();
+        let mut chars = text.code_points();
         chars.next(); // Consume the bang
         chars.next(); // Consume one r,s,a char
-        (Some(conversion), chars.as_str())
+        (Some(conversion), chars.as_wtf8())
     }
 }
 
 impl FormatConversion {
-    pub fn from_char(c: char) -> Option<FormatConversion> {
-        match c {
+    pub fn from_char(c: CodePoint) -> Option<FormatConversion> {
+        match c.to_char_lossy() {
             's' => Some(FormatConversion::Str),
             'r' => Some(FormatConversion::Repr),
             'a' => Some(FormatConversion::Ascii),
@@ -45,9 +47,9 @@ impl FormatConversion {
         }
     }
 
-    fn from_string(text: &str) -> Option<FormatConversion> {
-        let mut chars = text.chars();
-        if chars.next() != Some('!') {
+    fn from_string(text: &Wtf8) -> Option<FormatConversion> {
+        let mut chars = text.code_points();
+        if chars.next()? != '!' {
             return None;
         }
 
@@ -64,8 +66,8 @@ pub enum FormatAlign {
 }
 
 impl FormatAlign {
-    fn from_char(c: char) -> Option<FormatAlign> {
-        match c {
+    fn from_char(c: CodePoint) -> Option<FormatAlign> {
+        match c.to_char_lossy() {
             '<' => Some(FormatAlign::Left),
             '>' => Some(FormatAlign::Right),
             '=' => Some(FormatAlign::AfterSign),
@@ -76,10 +78,10 @@ impl FormatAlign {
 }
 
 impl FormatParse for FormatAlign {
-    fn parse(text: &str) -> (Option<Self>, &str) {
-        let mut chars = text.chars();
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
+        let mut chars = text.code_points();
         if let Some(maybe_align) = chars.next().and_then(Self::from_char) {
-            (Some(maybe_align), chars.as_str())
+            (Some(maybe_align), chars.as_wtf8())
         } else {
             (None, text)
         }
@@ -94,12 +96,12 @@ pub enum FormatSign {
 }
 
 impl FormatParse for FormatSign {
-    fn parse(text: &str) -> (Option<Self>, &str) {
-        let mut chars = text.chars();
-        match chars.next() {
-            Some('-') => (Some(Self::Minus), chars.as_str()),
-            Some('+') => (Some(Self::Plus), chars.as_str()),
-            Some(' ') => (Some(Self::MinusOrSpace), chars.as_str()),
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
+        let mut chars = text.code_points();
+        match chars.next().and_then(CodePoint::to_char) {
+            Some('-') => (Some(Self::Minus), chars.as_wtf8()),
+            Some('+') => (Some(Self::Plus), chars.as_wtf8()),
+            Some(' ') => (Some(Self::MinusOrSpace), chars.as_wtf8()),
             _ => (None, text),
         }
     }
@@ -112,11 +114,11 @@ pub enum FormatGrouping {
 }
 
 impl FormatParse for FormatGrouping {
-    fn parse(text: &str) -> (Option<Self>, &str) {
-        let mut chars = text.chars();
-        match chars.next() {
-            Some('_') => (Some(Self::Underscore), chars.as_str()),
-            Some(',') => (Some(Self::Comma), chars.as_str()),
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
+        let mut chars = text.code_points();
+        match chars.next().and_then(CodePoint::to_char) {
+            Some('_') => (Some(Self::Underscore), chars.as_wtf8()),
+            Some(',') => (Some(Self::Comma), chars.as_wtf8()),
             _ => (None, text),
         }
     }
@@ -161,25 +163,25 @@ impl From<&FormatType> for char {
 }
 
 impl FormatParse for FormatType {
-    fn parse(text: &str) -> (Option<Self>, &str) {
-        let mut chars = text.chars();
-        match chars.next() {
-            Some('s') => (Some(Self::String), chars.as_str()),
-            Some('b') => (Some(Self::Binary), chars.as_str()),
-            Some('c') => (Some(Self::Character), chars.as_str()),
-            Some('d') => (Some(Self::Decimal), chars.as_str()),
-            Some('o') => (Some(Self::Octal), chars.as_str()),
-            Some('n') => (Some(Self::Number(Case::Lower)), chars.as_str()),
-            Some('N') => (Some(Self::Number(Case::Upper)), chars.as_str()),
-            Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_str()),
-            Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_str()),
-            Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_str()),
-            Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_str()),
-            Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_str()),
-            Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_str()),
-            Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_str()),
-            Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_str()),
-            Some('%') => (Some(Self::Percentage), chars.as_str()),
+    fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
+        let mut chars = text.code_points();
+        match chars.next().and_then(CodePoint::to_char) {
+            Some('s') => (Some(Self::String), chars.as_wtf8()),
+            Some('b') => (Some(Self::Binary), chars.as_wtf8()),
+            Some('c') => (Some(Self::Character), chars.as_wtf8()),
+            Some('d') => (Some(Self::Decimal), chars.as_wtf8()),
+            Some('o') => (Some(Self::Octal), chars.as_wtf8()),
+            Some('n') => (Some(Self::Number(Case::Lower)), chars.as_wtf8()),
+            Some('N') => (Some(Self::Number(Case::Upper)), chars.as_wtf8()),
+            Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_wtf8()),
+            Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_wtf8()),
+            Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_wtf8()),
+            Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_wtf8()),
+            Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_wtf8()),
+            Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_wtf8()),
+            Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_wtf8()),
+            Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_wtf8()),
+            Some('%') => (Some(Self::Percentage), chars.as_wtf8()),
             _ => (None, text),
         }
     }
@@ -188,7 +190,7 @@ impl FormatParse for FormatType {
 #[derive(Debug, PartialEq)]
 pub struct FormatSpec {
     conversion: Option<FormatConversion>,
-    fill: Option<char>,
+    fill: Option<CodePoint>,
     align: Option<FormatAlign>,
     sign: Option<FormatSign>,
     alternate_form: bool,
@@ -198,17 +200,17 @@ pub struct FormatSpec {
     format_type: Option<FormatType>,
 }
 
-fn get_num_digits(text: &str) -> usize {
-    for (index, character) in text.char_indices() {
-        if !character.is_ascii_digit() {
+fn get_num_digits(text: &Wtf8) -> usize {
+    for (index, character) in text.code_point_indices() {
+        if !character.is_char_and(|c| c.is_ascii_digit()) {
             return index;
         }
     }
     text.len()
 }
 
-fn parse_fill_and_align(text: &str) -> (Option<char>, Option<FormatAlign>, &str) {
-    let char_indices: Vec<(usize, char)> = text.char_indices().take(3).collect();
+fn parse_fill_and_align(text: &Wtf8) -> (Option<CodePoint>, Option<FormatAlign>, &Wtf8) {
+    let char_indices: Vec<(usize, CodePoint)> = text.code_point_indices().take(3).collect();
     if char_indices.is_empty() {
         (None, None, text)
     } else if char_indices.len() == 1 {
@@ -225,12 +227,12 @@ fn parse_fill_and_align(text: &str) -> (Option<char>, Option<FormatAlign>, &str)
     }
 }
 
-fn parse_number(text: &str) -> Result<(Option<usize>, &str), FormatSpecError> {
+fn parse_number(text: &Wtf8) -> Result<(Option<usize>, &Wtf8), FormatSpecError> {
     let num_digits: usize = get_num_digits(text);
     if num_digits == 0 {
         return Ok((None, text));
     }
-    if let Ok(num) = text[..num_digits].parse::<usize>() {
+    if let Some(num) = parse_usize(&text[..num_digits]) {
         Ok((Some(num), &text[num_digits..]))
     } else {
         // NOTE: this condition is different from CPython
@@ -238,27 +240,27 @@ fn parse_number(text: &str) -> Result<(Option<usize>, &str), FormatSpecError> {
     }
 }
 
-fn parse_alternate_form(text: &str) -> (bool, &str) {
-    let mut chars = text.chars();
-    match chars.next() {
-        Some('#') => (true, chars.as_str()),
+fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) {
+    let mut chars = text.code_points();
+    match chars.next().and_then(CodePoint::to_char) {
+        Some('#') => (true, chars.as_wtf8()),
         _ => (false, text),
     }
 }
 
-fn parse_zero(text: &str) -> (bool, &str) {
-    let mut chars = text.chars();
-    match chars.next() {
-        Some('0') => (true, chars.as_str()),
+fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) {
+    let mut chars = text.code_points();
+    match chars.next().and_then(CodePoint::to_char) {
+        Some('0') => (true, chars.as_wtf8()),
         _ => (false, text),
     }
 }
 
-fn parse_precision(text: &str) -> Result<(Option<usize>, &str), FormatSpecError> {
-    let mut chars = text.chars();
-    Ok(match chars.next() {
+fn parse_precision(text: &Wtf8) -> Result<(Option<usize>, &Wtf8), FormatSpecError> {
+    let mut chars = text.code_points();
+    Ok(match chars.next().and_then(CodePoint::to_char) {
         Some('.') => {
-            let (size, remaining) = parse_number(chars.as_str())?;
+            let (size, remaining) = parse_number(chars.as_wtf8())?;
             if let Some(size) = size {
                 if size > i32::MAX as usize {
                     return Err(FormatSpecError::PrecisionTooBig);
@@ -273,7 +275,10 @@ fn parse_precision(text: &str) -> Result<(Option<usize>, &str), FormatSpecError>
 }
 
 impl FormatSpec {
-    pub fn parse(text: &str) -> Result<Self, FormatSpecError> {
+    pub fn parse(text: impl AsRef<Wtf8>) -> Result<Self, FormatSpecError> {
+        Self::_parse(text.as_ref())
+    }
+    fn _parse(text: &Wtf8) -> Result<Self, FormatSpecError> {
         // get_integer in CPython
         let (conversion, text) = FormatConversion::parse(text);
         let (mut fill, mut align, text) = parse_fill_and_align(text);
@@ -289,7 +294,7 @@ impl FormatSpec {
         }
 
         if zero && fill.is_none() {
-            fill.replace('0');
+            fill.replace('0'.into());
             align = align.or(Some(FormatAlign::AfterSign));
         }
 
@@ -306,10 +311,8 @@ impl FormatSpec {
         })
     }
 
-    fn compute_fill_string(fill_char: char, fill_chars_needed: i32) -> String {
-        (0..fill_chars_needed)
-            .map(|_| fill_char)
-            .collect::<String>()
+    fn compute_fill_string(fill_char: CodePoint, fill_chars_needed: i32) -> Wtf8Buf {
+        (0..fill_chars_needed).map(|_| fill_char).collect()
     }
 
     fn add_magnitude_separators_for_char(
@@ -625,7 +628,7 @@ impl FormatSpec {
         let align = self.align.unwrap_or(default_align);
 
         let num_chars = magnitude_str.char_len();
-        let fill_char = self.fill.unwrap_or(' ');
+        let fill_char = self.fill.unwrap_or(' '.into());
         let fill_chars_needed: i32 = self.width.map_or(0, |w| {
             cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32))
         });
@@ -726,20 +729,20 @@ impl FromStr for FormatSpec {
 
 #[derive(Debug, PartialEq)]
 pub enum FieldNamePart {
-    Attribute(String),
+    Attribute(Wtf8Buf),
     Index(usize),
-    StringIndex(String),
+    StringIndex(Wtf8Buf),
 }
 
 impl FieldNamePart {
     fn parse_part(
-        chars: &mut impl PeekingNext<Item = char>,
+        chars: &mut impl PeekingNext<Item = CodePoint>,
     ) -> Result<Option<FieldNamePart>, FormatParseError> {
         chars
             .next()
-            .map(|ch| match ch {
+            .map(|ch| match ch.to_char_lossy() {
                 '.' => {
-                    let mut attribute = String::new();
+                    let mut attribute = Wtf8Buf::new();
                     for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') {
                         attribute.push(ch);
                     }
@@ -750,12 +753,12 @@ impl FieldNamePart {
                     }
                 }
                 '[' => {
-                    let mut index = String::new();
+                    let mut index = Wtf8Buf::new();
                     for ch in chars {
                         if ch == ']' {
                             return if index.is_empty() {
                                 Err(FormatParseError::EmptyAttribute)
-                            } else if let Ok(index) = index.parse::<usize>() {
+                            } else if let Some(index) = parse_usize(&index) {
                                 Ok(FieldNamePart::Index(index))
                             } else {
                                 Ok(FieldNamePart::StringIndex(index))
@@ -775,7 +778,7 @@ impl FieldNamePart {
 pub enum FieldType {
     Auto,
     Index(usize),
-    Keyword(String),
+    Keyword(Wtf8Buf),
 }
 
 #[derive(Debug, PartialEq)]
@@ -784,17 +787,20 @@ pub struct FieldName {
     pub parts: Vec<FieldNamePart>,
 }
 
+fn parse_usize(s: &Wtf8) -> Option<usize> {
+    s.as_str().ok().and_then(|s| s.parse().ok())
+}
+
 impl FieldName {
-    pub fn parse(text: &str) -> Result<FieldName, FormatParseError> {
-        let mut chars = text.chars().peekable();
-        let mut first = String::new();
-        for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') {
-            first.push(ch);
-        }
+    pub fn parse(text: &Wtf8) -> Result<FieldName, FormatParseError> {
+        let mut chars = text.code_points().peekable();
+        let first: Wtf8Buf = chars
+            .peeking_take_while(|ch| *ch != '.' && *ch != '[')
+            .collect();
 
         let field_type = if first.is_empty() {
             FieldType::Auto
-        } else if let Ok(index) = first.parse::<usize>() {
+        } else if let Some(index) = parse_usize(&first) {
             FieldType::Index(index)
         } else {
             FieldType::Keyword(first)
@@ -812,11 +818,11 @@ impl FieldName {
 #[derive(Debug, PartialEq)]
 pub enum FormatPart {
     Field {
-        field_name: String,
-        conversion_spec: Option<char>,
-        format_spec: String,
+        field_name: Wtf8Buf,
+        conversion_spec: Option<CodePoint>,
+        format_spec: Wtf8Buf,
     },
-    Literal(String),
+    Literal(Wtf8Buf),
 }
 
 #[derive(Debug, PartialEq)]
@@ -825,8 +831,8 @@ pub struct FormatString {
 }
 
 impl FormatString {
-    fn parse_literal_single(text: &str) -> Result<(char, &str), FormatParseError> {
-        let mut chars = text.chars();
+    fn parse_literal_single(text: &Wtf8) -> Result<(CodePoint, &Wtf8), FormatParseError> {
+        let mut chars = text.code_points();
         // This should never be called with an empty str
         let first_char = chars.next().unwrap();
         // isn't this detectable only with bytes operation?
@@ -836,15 +842,15 @@ impl FormatString {
             return if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char {
                 Err(FormatParseError::UnescapedStartBracketInLiteral)
             } else {
-                Ok((first_char, chars.as_str()))
+                Ok((first_char, chars.as_wtf8()))
             };
         }
-        Ok((first_char, chars.as_str()))
+        Ok((first_char, chars.as_wtf8()))
     }
 
-    fn parse_literal(text: &str) -> Result<(FormatPart, &str), FormatParseError> {
+    fn parse_literal(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> {
         let mut cur_text = text;
-        let mut result_string = String::new();
+        let mut result_string = Wtf8Buf::new();
         while !cur_text.is_empty() {
             match FormatString::parse_literal_single(cur_text) {
                 Ok((next_char, remaining)) => {
@@ -860,14 +866,14 @@ impl FormatString {
                 }
             }
         }
-        Ok((FormatPart::Literal(result_string), ""))
+        Ok((FormatPart::Literal(result_string), "".as_ref()))
     }
 
-    fn parse_part_in_brackets(text: &str) -> Result<FormatPart, FormatParseError> {
-        let mut chars = text.chars().peekable();
+    fn parse_part_in_brackets(text: &Wtf8) -> Result<FormatPart, FormatParseError> {
+        let mut chars = text.code_points().peekable();
 
-        let mut left = String::new();
-        let mut right = String::new();
+        let mut left = Wtf8Buf::new();
+        let mut right = Wtf8Buf::new();
 
         let mut split = false;
         let mut selected = &mut left;
@@ -899,12 +905,12 @@ impl FormatString {
         }
 
         // before the comma is a keyword or arg index, after the comma is maybe a spec.
-        let arg_part: &str = &left;
+        let arg_part: &Wtf8 = &left;
 
-        let format_spec = if split { right } else { String::new() };
+        let format_spec = if split { right } else { Wtf8Buf::new() };
 
         // left can still be the conversion (!r, !s, !a)
-        let parts: Vec<&str> = arg_part.splitn(2, '!').collect();
+        let parts: Vec<&Wtf8> = arg_part.splitn(2, "!".as_ref()).collect();
         // before the bang is a keyword or arg index, after the comma is maybe a conversion spec.
         let arg_part = parts[0];
 
@@ -913,7 +919,7 @@ impl FormatString {
             .map(|conversion| {
                 // conversions are only every one character
                 conversion
-                    .chars()
+                    .code_points()
                     .exactly_one()
                     .map_err(|_| FormatParseError::UnknownConversion)
             })
@@ -926,13 +932,13 @@ impl FormatString {
         })
     }
 
-    fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> {
+    fn parse_spec(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> {
         let mut nested = false;
         let mut end_bracket_pos = None;
-        let mut left = String::new();
+        let mut left = Wtf8Buf::new();
 
         // There may be one layer nesting brackets in spec
-        for (idx, c) in text.char_indices() {
+        for (idx, c) in text.code_point_indices() {
             if idx == 0 {
                 if c != '{' {
                     return Err(FormatParseError::MissingStartBracket);
@@ -959,7 +965,7 @@ impl FormatString {
             }
         }
         if let Some(pos) = end_bracket_pos {
-            let (_, right) = text.split_at(pos);
+            let right = &text[pos..];
             let format_part = FormatString::parse_part_in_brackets(&left)?;
             Ok((format_part, &right[1..]))
         } else {
@@ -970,14 +976,14 @@ impl FormatString {
 
 pub trait FromTemplate<'a>: Sized {
     type Err;
-    fn from_str(s: &'a str) -> Result<Self, Self::Err>;
+    fn from_str(s: &'a Wtf8) -> Result<Self, Self::Err>;
 }
 
 impl<'a> FromTemplate<'a> for FormatString {
     type Err = FormatParseError;
 
-    fn from_str(text: &'a str) -> Result<Self, Self::Err> {
-        let mut cur_text: &str = text;
+    fn from_str(text: &'a Wtf8) -> Result<Self, Self::Err> {
+        let mut cur_text: &Wtf8 = text;
         let mut parts: Vec<FormatPart> = Vec::new();
         while !cur_text.is_empty() {
             // Try to parse both literals and bracketed format parts until we
@@ -1001,6 +1007,14 @@ mod tests {
 
     #[test]
     fn test_fill_and_align() {
+        let parse_fill_and_align = |text| {
+            let (fill, align, rest) = parse_fill_and_align(str::as_ref(text));
+            (
+                fill.and_then(CodePoint::to_char),
+                align,
+                rest.as_str().unwrap(),
+            )
+        };
         assert_eq!(
             parse_fill_and_align(" <"),
             (Some(' '), Some(FormatAlign::Left), "")
@@ -1043,7 +1057,7 @@ mod tests {
     fn test_fill_and_width() {
         let expected = Ok(FormatSpec {
             conversion: None,
-            fill: Some('<'),
+            fill: Some('<'.into()),
             align: Some(FormatAlign::Right),
             sign: None,
             alternate_form: false,
@@ -1059,7 +1073,7 @@ mod tests {
     fn test_all() {
         let expected = Ok(FormatSpec {
             conversion: None,
-            fill: Some('<'),
+            fill: Some('<'.into()),
             align: Some(FormatAlign::Right),
             sign: Some(FormatSign::Minus),
             alternate_form: true,
@@ -1167,33 +1181,33 @@ mod tests {
     fn test_format_parse() {
         let expected = Ok(FormatString {
             format_parts: vec![
-                FormatPart::Literal("abcd".to_owned()),
+                FormatPart::Literal("abcd".into()),
                 FormatPart::Field {
-                    field_name: "1".to_owned(),
+                    field_name: "1".into(),
                     conversion_spec: None,
-                    format_spec: String::new(),
+                    format_spec: "".into(),
                 },
-                FormatPart::Literal(":".to_owned()),
+                FormatPart::Literal(":".into()),
                 FormatPart::Field {
-                    field_name: "key".to_owned(),
+                    field_name: "key".into(),
                     conversion_spec: None,
-                    format_spec: String::new(),
+                    format_spec: "".into(),
                 },
             ],
         });
 
-        assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected);
+        assert_eq!(FormatString::from_str("abcd{1}:{key}".as_ref()), expected);
     }
 
     #[test]
     fn test_format_parse_multi_byte_char() {
-        assert!(FormatString::from_str("{a:%ЫйЯЧ}").is_ok());
+        assert!(FormatString::from_str("{a:%ЫйЯЧ}".as_ref()).is_ok());
     }
 
     #[test]
     fn test_format_parse_fail() {
         assert_eq!(
-            FormatString::from_str("{s"),
+            FormatString::from_str("{s".as_ref()),
             Err(FormatParseError::UnmatchedBracket)
         );
     }
@@ -1201,27 +1215,27 @@ mod tests {
     #[test]
     fn test_square_brackets_inside_format() {
         assert_eq!(
-            FormatString::from_str("{[:123]}"),
+            FormatString::from_str("{[:123]}".as_ref()),
             Ok(FormatString {
                 format_parts: vec![FormatPart::Field {
-                    field_name: "[:123]".to_owned(),
+                    field_name: "[:123]".into(),
                     conversion_spec: None,
-                    format_spec: "".to_owned(),
+                    format_spec: "".into(),
                 }],
             }),
         );
 
-        assert_eq!(FormatString::from_str("{asdf[:123]asdf}"), {
+        assert_eq!(FormatString::from_str("{asdf[:123]asdf}".as_ref()), {
             Ok(FormatString {
                 format_parts: vec![FormatPart::Field {
-                    field_name: "asdf[:123]asdf".to_owned(),
+                    field_name: "asdf[:123]asdf".into(),
                     conversion_spec: None,
-                    format_spec: "".to_owned(),
+                    format_spec: "".into(),
                 }],
             })
         });
 
-        assert_eq!(FormatString::from_str("{[1234}"), {
+        assert_eq!(FormatString::from_str("{[1234}".as_ref()), {
             Err(FormatParseError::MissingRightBracket)
         });
     }
@@ -1230,17 +1244,17 @@ mod tests {
     fn test_format_parse_escape() {
         let expected = Ok(FormatString {
             format_parts: vec![
-                FormatPart::Literal("{".to_owned()),
+                FormatPart::Literal("{".into()),
                 FormatPart::Field {
-                    field_name: "key".to_owned(),
+                    field_name: "key".into(),
                     conversion_spec: None,
-                    format_spec: String::new(),
+                    format_spec: "".into(),
                 },
-                FormatPart::Literal("}ddfe".to_owned()),
+                FormatPart::Literal("}ddfe".into()),
             ],
         });
 
-        assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected);
+        assert_eq!(FormatString::from_str("{{{key}}}ddfe".as_ref()), expected);
     }
 
     #[test]
@@ -1277,52 +1291,44 @@ mod tests {
 
     #[test]
     fn test_parse_field_name() {
+        let parse = |s: &str| FieldName::parse(s.as_ref());
         assert_eq!(
-            FieldName::parse(""),
+            parse(""),
             Ok(FieldName {
                 field_type: FieldType::Auto,
                 parts: Vec::new(),
             })
         );
         assert_eq!(
-            FieldName::parse("0"),
+            parse("0"),
             Ok(FieldName {
                 field_type: FieldType::Index(0),
                 parts: Vec::new(),
             })
         );
         assert_eq!(
-            FieldName::parse("key"),
+            parse("key"),
             Ok(FieldName {
-                field_type: FieldType::Keyword("key".to_owned()),
+                field_type: FieldType::Keyword("key".into()),
                 parts: Vec::new(),
             })
         );
         assert_eq!(
-            FieldName::parse("key.attr[0][string]"),
+            parse("key.attr[0][string]"),
             Ok(FieldName {
-                field_type: FieldType::Keyword("key".to_owned()),
+                field_type: FieldType::Keyword("key".into()),
                 parts: vec![
-                    FieldNamePart::Attribute("attr".to_owned()),
+                    FieldNamePart::Attribute("attr".into()),
                     FieldNamePart::Index(0),
-                    FieldNamePart::StringIndex("string".to_owned())
+                    FieldNamePart::StringIndex("string".into())
                 ],
             })
         );
+        assert_eq!(parse("key.."), Err(FormatParseError::EmptyAttribute));
+        assert_eq!(parse("key[]"), Err(FormatParseError::EmptyAttribute));
+        assert_eq!(parse("key["), Err(FormatParseError::MissingRightBracket));
         assert_eq!(
-            FieldName::parse("key.."),
-            Err(FormatParseError::EmptyAttribute)
-        );
-        assert_eq!(
-            FieldName::parse("key[]"),
-            Err(FormatParseError::EmptyAttribute)
-        );
-        assert_eq!(
-            FieldName::parse("key["),
-            Err(FormatParseError::MissingRightBracket)
-        );
-        assert_eq!(
-            FieldName::parse("key[0]after"),
+            parse("key[0]after"),
             Err(FormatParseError::InvalidCharacterAfterRightBracket)
         );
     }

common/src/lib.rs

@@ -29,6 +29,7 @@ pub mod static_cell;
 pub mod str;
 #[cfg(windows)]
 pub mod windows;
+pub mod wtf8;
 
 pub mod vendored {
     pub use ascii;

common/src/str.rs

@@ -1,9 +1,9 @@
-use crate::{
-    atomic::{PyAtomic, Radium},
-    format::CharLen,
-    hash::PyHash,
-};
-use ascii::AsciiString;
+use crate::atomic::{PyAtomic, Radium};
+use crate::format::CharLen;
+use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
+use ascii::{AsciiChar, AsciiStr, AsciiString};
+use core::fmt;
+use core::sync::atomic::Ordering::Relaxed;
 use std::ops::{Bound, RangeBounds};
 
 #[cfg(not(target_arch = "wasm32"))]
@@ -14,133 +14,323 @@ pub type wchar_t = libc::wchar_t;
 pub type wchar_t = u32;
 
 /// Utf8 + state.ascii (+ PyUnicode_Kind in future)
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub enum PyStrKind {
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum StrKind {
     Ascii,
     Utf8,
+    Wtf8,
 }
 
-impl std::ops::BitOr for PyStrKind {
+impl std::ops::BitOr for StrKind {
     type Output = Self;
     fn bitor(self, other: Self) -> Self {
+        use StrKind::*;
         match (self, other) {
-            (Self::Ascii, Self::Ascii) => Self::Ascii,
-            _ => Self::Utf8,
+            (Wtf8, _) | (_, Wtf8) => Wtf8,
+            (Utf8, _) | (_, Utf8) => Utf8,
+            (Ascii, Ascii) => Ascii,
         }
     }
 }
 
-impl PyStrKind {
-    #[inline]
-    pub fn new_data(self) -> PyStrKindData {
+impl StrKind {
+    pub fn is_ascii(&self) -> bool {
+        matches!(self, Self::Ascii)
+    }
+
+    pub fn is_utf8(&self) -> bool {
+        matches!(self, Self::Ascii | Self::Utf8)
+    }
+
+    #[inline(always)]
+    pub fn can_encode(&self, code: CodePoint) -> bool {
         match self {
-            PyStrKind::Ascii => PyStrKindData::Ascii,
-            PyStrKind::Utf8 => PyStrKindData::Utf8(Radium::new(usize::MAX)),
+            StrKind::Ascii => code.is_ascii(),
+            StrKind::Utf8 => code.to_char().is_some(),
+            StrKind::Wtf8 => true,
         }
     }
 }
 
+pub trait DeduceStrKind {
+    fn str_kind(&self) -> StrKind;
+}
+
+impl DeduceStrKind for str {
+    fn str_kind(&self) -> StrKind {
+        if self.is_ascii() {
+            StrKind::Ascii
+        } else {
+            StrKind::Utf8
+        }
+    }
+}
+
+impl DeduceStrKind for Wtf8 {
+    fn str_kind(&self) -> StrKind {
+        if self.is_ascii() {
+            StrKind::Ascii
+        } else if self.is_utf8() {
+            StrKind::Utf8
+        } else {
+            StrKind::Wtf8
+        }
+    }
+}
+
+impl DeduceStrKind for String {
+    fn str_kind(&self) -> StrKind {
+        (**self).str_kind()
+    }
+}
+
+impl DeduceStrKind for Wtf8Buf {
+    fn str_kind(&self) -> StrKind {
+        (**self).str_kind()
+    }
+}
+
+impl<T: DeduceStrKind + ?Sized> DeduceStrKind for &T {
+    fn str_kind(&self) -> StrKind {
+        (**self).str_kind()
+    }
+}
+
+impl<T: DeduceStrKind + ?Sized> DeduceStrKind for Box<T> {
+    fn str_kind(&self) -> StrKind {
+        (**self).str_kind()
+    }
+}
+
 #[derive(Debug)]
-pub enum PyStrKindData {
-    Ascii,
-    // uses usize::MAX as a sentinel for "uncomputed"
-    Utf8(PyAtomic<usize>),
+pub enum PyKindStr<'a> {
+    Ascii(&'a AsciiStr),
+    Utf8(&'a str),
+    Wtf8(&'a Wtf8),
 }
 
-impl PyStrKindData {
-    #[inline]
-    pub fn kind(&self) -> PyStrKind {
-        match self {
-            PyStrKindData::Ascii => PyStrKind::Ascii,
-            PyStrKindData::Utf8(_) => PyStrKind::Utf8,
-        }
+#[derive(Debug, Clone)]
+pub struct StrData {
+    data: Box<Wtf8>,
+    kind: StrKind,
+    len: StrLen,
+}
+
+struct StrLen(PyAtomic<usize>);
+
+impl From<usize> for StrLen {
+    #[inline(always)]
+    fn from(value: usize) -> Self {
+        Self(Radium::new(value))
     }
 }
 
-pub struct BorrowedStr<'a> {
-    bytes: &'a [u8],
-    kind: PyStrKindData,
-    #[allow(dead_code)]
-    hash: PyAtomic<PyHash>,
-}
-
-impl<'a> BorrowedStr<'a> {
-    /// # Safety
-    /// `s` have to be an ascii string
-    #[inline]
-    pub unsafe fn from_ascii_unchecked(s: &'a [u8]) -> Self {
-        debug_assert!(s.is_ascii());
-        Self {
-            bytes: s,
-            kind: PyStrKind::Ascii.new_data(),
-            hash: PyAtomic::<PyHash>::new(0),
-        }
-    }
-
-    #[inline]
-    pub fn from_bytes(s: &'a [u8]) -> Self {
-        let k = if s.is_ascii() {
-            PyStrKind::Ascii.new_data()
+impl fmt::Debug for StrLen {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let len = self.0.load(Relaxed);
+        if len == usize::MAX {
+            f.write_str("<uncomputed>")
         } else {
-            PyStrKind::Utf8.new_data()
-        };
+            len.fmt(f)
+        }
+    }
+}
+
+impl StrLen {
+    #[inline(always)]
+    fn zero() -> Self {
+        0usize.into()
+    }
+    #[inline(always)]
+    fn uncomputed() -> Self {
+        usize::MAX.into()
+    }
+}
+
+impl Clone for StrLen {
+    fn clone(&self) -> Self {
+        Self(self.0.load(Relaxed).into())
+    }
+}
+
+impl Default for StrData {
+    fn default() -> Self {
         Self {
-            bytes: s,
-            kind: k,
-            hash: PyAtomic::<PyHash>::new(0),
+            data: <Box<Wtf8>>::default(),
+            kind: StrKind::Ascii,
+            len: StrLen::zero(),
+        }
+    }
+}
+
+impl From<Box<Wtf8>> for StrData {
+    fn from(value: Box<Wtf8>) -> Self {
+        // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for
+        // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532
+        let kind = value.str_kind();
+        unsafe { Self::new_str_unchecked(value, kind) }
+    }
+}
+
+impl From<Box<str>> for StrData {
+    #[inline]
+    fn from(value: Box<str>) -> Self {
+        // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for
+        // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532
+        let kind = value.str_kind();
+        unsafe { Self::new_str_unchecked(value.into(), kind) }
+    }
+}
+
+impl From<Box<AsciiStr>> for StrData {
+    #[inline]
+    fn from(value: Box<AsciiStr>) -> Self {
+        Self {
+            len: value.len().into(),
+            data: value.into(),
+            kind: StrKind::Ascii,
+        }
+    }
+}
+
+impl From<AsciiChar> for StrData {
+    fn from(ch: AsciiChar) -> Self {
+        AsciiString::from(ch).into_boxed_ascii_str().into()
+    }
+}
+
+impl From<char> for StrData {
+    fn from(ch: char) -> Self {
+        if let Ok(ch) = ascii::AsciiChar::from_ascii(ch) {
+            ch.into()
+        } else {
+            Self {
+                data: ch.to_string().into(),
+                kind: StrKind::Utf8,
+                len: 1.into(),
+            }
+        }
+    }
+}
+
+impl From<CodePoint> for StrData {
+    fn from(ch: CodePoint) -> Self {
+        if let Some(ch) = ch.to_char() {
+            ch.into()
+        } else {
+            Self {
+                data: Wtf8Buf::from(ch).into(),
+                kind: StrKind::Wtf8,
+                len: 1.into(),
+            }
+        }
+    }
+}
+
+impl StrData {
+    /// # Safety
+    ///
+    /// Given `bytes` must be valid data for given `kind`
+    pub unsafe fn new_str_unchecked(data: Box<Wtf8>, kind: StrKind) -> Self {
+        let len = match kind {
+            StrKind::Ascii => data.len().into(),
+            _ => StrLen::uncomputed(),
+        };
+        Self { data, kind, len }
+    }
+
+    /// # Safety
+    ///
+    /// `char_len` must be accurate.
+    pub unsafe fn new_with_char_len(data: Box<Wtf8>, kind: StrKind, char_len: usize) -> Self {
+        Self {
+            data,
+            kind,
+            len: char_len.into(),
         }
     }
 
     #[inline]
-    pub fn as_str(&self) -> &str {
-        unsafe {
-            // SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string
-            std::str::from_utf8_unchecked(self.bytes)
+    pub fn as_wtf8(&self) -> &Wtf8 {
+        &self.data
+    }
+
+    #[inline]
+    pub fn as_str(&self) -> Option<&str> {
+        self.kind
+            .is_utf8()
+            .then(|| unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) })
+    }
+
+    pub fn as_ascii(&self) -> Option<&AsciiStr> {
+        self.kind
+            .is_ascii()
+            .then(|| unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) })
+    }
+
+    pub fn kind(&self) -> StrKind {
+        self.kind
+    }
+
+    #[inline]
+    pub fn as_str_kind(&self) -> PyKindStr<'_> {
+        match self.kind {
+            StrKind::Ascii => {
+                PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) })
+            }
+            StrKind::Utf8 => {
+                PyKindStr::Utf8(unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) })
+            }
+            StrKind::Wtf8 => PyKindStr::Wtf8(&self.data),
         }
     }
 
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.data.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.data.is_empty()
+    }
+
     #[inline]
     pub fn char_len(&self) -> usize {
-        match self.kind {
-            PyStrKindData::Ascii => self.bytes.len(),
-            PyStrKindData::Utf8(ref len) => match len.load(core::sync::atomic::Ordering::Relaxed) {
-                usize::MAX => self._compute_char_len(),
-                len => len,
-            },
+        match self.len.0.load(Relaxed) {
+            usize::MAX => self._compute_char_len(),
+            len => len,
         }
     }
 
     #[cold]
     fn _compute_char_len(&self) -> usize {
-        match self.kind {
-            PyStrKindData::Utf8(ref char_len) => {
-                let len = self.as_str().chars().count();
-                // len cannot be usize::MAX, since vec.capacity() < sys.maxsize
-                char_len.store(len, core::sync::atomic::Ordering::Relaxed);
-                len
-            }
-            _ => unsafe {
-                debug_assert!(false); // invalid for non-utf8 strings
-                std::hint::unreachable_unchecked()
-            },
+        let len = if let Some(s) = self.as_str() {
+            // utf8 chars().count() is optimized
+            s.chars().count()
+        } else {
+            self.data.code_points().count()
+        };
+        // len cannot be usize::MAX, since vec.capacity() < sys.maxsize
+        self.len.0.store(len, Relaxed);
+        len
+    }
+
+    pub fn nth_char(&self, index: usize) -> CodePoint {
+        match self.as_str_kind() {
+            PyKindStr::Ascii(s) => s[index].into(),
+            PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(),
+            PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(),
         }
     }
 }
 
-impl std::ops::Deref for BorrowedStr<'_> {
-    type Target = str;
-    fn deref(&self) -> &str {
-        self.as_str()
-    }
-}
-
-impl std::fmt::Display for BorrowedStr<'_> {
+impl std::fmt::Display for StrData {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.as_str().fmt(f)
+        self.data.fmt(f)
     }
 }
 
-impl CharLen for BorrowedStr<'_> {
+impl CharLen for StrData {
     fn char_len(&self) -> usize {
         self.char_len()
     }
@@ -181,6 +371,41 @@ pub fn char_range_end(s: &str, nchars: usize) -> Option<usize> {
     Some(i)
 }
 
+pub fn try_get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> Option<&Wtf8> {
+    let mut chars = w.code_points();
+    let start = match range.start_bound() {
+        Bound::Included(&i) => i,
+        Bound::Excluded(&i) => i + 1,
+        Bound::Unbounded => 0,
+    };
+    for _ in 0..start {
+        chars.next()?;
+    }
+    let s = chars.as_wtf8();
+    let range_len = match range.end_bound() {
+        Bound::Included(&i) => i + 1 - start,
+        Bound::Excluded(&i) => i - start,
+        Bound::Unbounded => return Some(s),
+    };
+    codepoint_range_end(s, range_len).map(|end| &s[..end])
+}
+
+pub fn get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> &Wtf8 {
+    try_get_codepoints(w, range).unwrap()
+}
+
+#[inline]
+pub fn codepoint_range_end(s: &Wtf8, nchars: usize) -> Option<usize> {
+    let i = match nchars.checked_sub(1) {
+        Some(last_char_index) => {
+            let (index, c) = s.code_point_indices().nth(last_char_index)?;
+            index + c.len_wtf8()
+        }
+        None => 0,
+    };
+    Some(i)
+}
+
 pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> {
     if width <= bytes.len() {
         bytes.to_vec()

common/src/wtf8/core_char.rs

@@ -0,0 +1,113 @@
+//! Unstable functions from [`core::char`]
+
+use core::slice;
+
+pub const MAX_LEN_UTF8: usize = 4;
+pub const MAX_LEN_UTF16: usize = 2;
+
+// UTF-8 ranges and tags for encoding characters
+const TAG_CONT: u8 = 0b1000_0000;
+const TAG_TWO_B: u8 = 0b1100_0000;
+const TAG_THREE_B: u8 = 0b1110_0000;
+const TAG_FOUR_B: u8 = 0b1111_0000;
+const MAX_ONE_B: u32 = 0x80;
+const MAX_TWO_B: u32 = 0x800;
+const MAX_THREE_B: u32 = 0x10000;
+
+#[inline]
+#[must_use]
+pub const fn len_utf8(code: u32) -> usize {
+    match code {
+        ..MAX_ONE_B => 1,
+        ..MAX_TWO_B => 2,
+        ..MAX_THREE_B => 3,
+        _ => 4,
+    }
+}
+
+#[inline]
+#[must_use]
+const fn len_utf16(code: u32) -> usize {
+    if (code & 0xFFFF) == code { 1 } else { 2 }
+}
+
+/// Encodes a raw `u32` value as UTF-8 into the provided byte buffer,
+/// and then returns the subslice of the buffer that contains the encoded character.
+///
+/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range.
+/// (Creating a `char` in the surrogate range is UB.)
+/// The result is valid [generalized UTF-8] but not valid UTF-8.
+///
+/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8
+///
+/// # Panics
+///
+/// Panics if the buffer is not large enough.
+/// A buffer of length four is large enough to encode any `char`.
+#[doc(hidden)]
+#[inline]
+pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] {
+    let len = len_utf8(code);
+    match (len, &mut *dst) {
+        (1, [a, ..]) => {
+            *a = code as u8;
+        }
+        (2, [a, b, ..]) => {
+            *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
+            *b = (code & 0x3F) as u8 | TAG_CONT;
+        }
+        (3, [a, b, c, ..]) => {
+            *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
+            *b = (code >> 6 & 0x3F) as u8 | TAG_CONT;
+            *c = (code & 0x3F) as u8 | TAG_CONT;
+        }
+        (4, [a, b, c, d, ..]) => {
+            *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
+            *b = (code >> 12 & 0x3F) as u8 | TAG_CONT;
+            *c = (code >> 6 & 0x3F) as u8 | TAG_CONT;
+            *d = (code & 0x3F) as u8 | TAG_CONT;
+        }
+        _ => {
+            panic!(
+                "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}",
+                dst_len = dst.len(),
+            )
+        }
+    };
+    // SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds.
+    unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) }
+}
+
+/// Encodes a raw `u32` value as UTF-16 into the provided `u16` buffer,
+/// and then returns the subslice of the buffer that contains the encoded character.
+///
+/// Unlike `char::encode_utf16`, this method also handles codepoints in the surrogate range.
+/// (Creating a `char` in the surrogate range is UB.)
+///
+/// # Panics
+///
+/// Panics if the buffer is not large enough.
+/// A buffer of length 2 is large enough to encode any `char`.
+#[doc(hidden)]
+#[inline]
+pub fn encode_utf16_raw(mut code: u32, dst: &mut [u16]) -> &mut [u16] {
+    let len = len_utf16(code);
+    match (len, &mut *dst) {
+        (1, [a, ..]) => {
+            *a = code as u16;
+        }
+        (2, [a, b, ..]) => {
+            code -= 0x1_0000;
+            *a = (code >> 10) as u16 | 0xD800;
+            *b = (code & 0x3FF) as u16 | 0xDC00;
+        }
+        _ => {
+            panic!(
+                "encode_utf16: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}",
+                dst_len = dst.len(),
+            )
+        }
+    };
+    // SAFETY: `<&mut [u16]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds.
+    unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) }
+}

common/src/wtf8/core_str.rs

@@ -0,0 +1,113 @@
+//! Operations related to UTF-8 validation.
+//!
+//! Copied from `core::str::validations`
+
+/// Returns the initial codepoint accumulator for the first byte.
+/// The first byte is special, only want bottom 5 bits for width 2, 4 bits
+/// for width 3, and 3 bits for width 4.
+#[inline]
+const fn utf8_first_byte(byte: u8, width: u32) -> u32 {
+    (byte & (0x7F >> width)) as u32
+}
+
+/// Returns the value of `ch` updated with continuation byte `byte`.
+#[inline]
+const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 {
+    (ch << 6) | (byte & CONT_MASK) as u32
+}
+
+/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the
+/// bits `10`).
+#[inline]
+pub(super) const fn utf8_is_cont_byte(byte: u8) -> bool {
+    (byte as i8) < -64
+}
+
+/// Reads the next code point out of a byte iterator (assuming a
+/// UTF-8-like encoding).
+///
+/// # Safety
+///
+/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string
+#[inline]
+pub unsafe fn next_code_point<'a, I: Iterator<Item = &'a u8>>(bytes: &mut I) -> Option<u32> {
+    // Decode UTF-8
+    let x = *bytes.next()?;
+    if x < 128 {
+        return Some(x as u32);
+    }
+
+    // Multibyte case follows
+    // Decode from a byte combination out of: [[[x y] z] w]
+    // NOTE: Performance is sensitive to the exact formulation here
+    let init = utf8_first_byte(x, 2);
+    // SAFETY: `bytes` produces an UTF-8-like string,
+    // so the iterator must produce a value here.
+    let y = unsafe { *bytes.next().unwrap_unchecked() };
+    let mut ch = utf8_acc_cont_byte(init, y);
+    if x >= 0xE0 {
+        // [[x y z] w] case
+        // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid
+        // SAFETY: `bytes` produces an UTF-8-like string,
+        // so the iterator must produce a value here.
+        let z = unsafe { *bytes.next().unwrap_unchecked() };
+        let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z);
+        ch = init << 12 | y_z;
+        if x >= 0xF0 {
+            // [x y z w] case
+            // use only the lower 3 bits of `init`
+            // SAFETY: `bytes` produces an UTF-8-like string,
+            // so the iterator must produce a value here.
+            let w = unsafe { *bytes.next().unwrap_unchecked() };
+            ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w);
+        }
+    }
+
+    Some(ch)
+}
+
+/// Reads the last code point out of a byte iterator (assuming a
+/// UTF-8-like encoding).
+///
+/// # Safety
+///
+/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string
+#[inline]
+pub unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option<u32>
+where
+    I: DoubleEndedIterator<Item = &'a u8>,
+{
+    // Decode UTF-8
+    let w = match *bytes.next_back()? {
+        next_byte if next_byte < 128 => return Some(next_byte as u32),
+        back_byte => back_byte,
+    };
+
+    // Multibyte case follows
+    // Decode from a byte combination out of: [x [y [z w]]]
+    let mut ch;
+    // SAFETY: `bytes` produces an UTF-8-like string,
+    // so the iterator must produce a value here.
+    let z = unsafe { *bytes.next_back().unwrap_unchecked() };
+    ch = utf8_first_byte(z, 2);
+    if utf8_is_cont_byte(z) {
+        // SAFETY: `bytes` produces an UTF-8-like string,
+        // so the iterator must produce a value here.
+        let y = unsafe { *bytes.next_back().unwrap_unchecked() };
+        ch = utf8_first_byte(y, 3);
+        if utf8_is_cont_byte(y) {
+            // SAFETY: `bytes` produces an UTF-8-like string,
+            // so the iterator must produce a value here.
+            let x = unsafe { *bytes.next_back().unwrap_unchecked() };
+            ch = utf8_first_byte(x, 4);
+            ch = utf8_acc_cont_byte(ch, y);
+        }
+        ch = utf8_acc_cont_byte(ch, z);
+    }
+    ch = utf8_acc_cont_byte(ch, w);
+
+    Some(ch)
+}
+
+/// Mask of the value bits of a continuation byte.
+const CONT_MASK: u8 = 0b0011_1111;

common/src/wtf8/core_str_count.rs

@@ -0,0 +1,161 @@
+//! Modified from core::str::count
+
+use super::Wtf8;
+
+const USIZE_SIZE: usize = core::mem::size_of::<usize>();
+const UNROLL_INNER: usize = 4;
+
+#[inline]
+pub(super) fn count_chars(s: &Wtf8) -> usize {
+    if s.len() < USIZE_SIZE * UNROLL_INNER {
+        // Avoid entering the optimized implementation for strings where the
+        // difference is not likely to matter, or where it might even be slower.
+        // That said, a ton of thought was not spent on the particular threshold
+        // here, beyond "this value seems to make sense".
+        char_count_general_case(s.as_bytes())
+    } else {
+        do_count_chars(s)
+    }
+}
+
+fn do_count_chars(s: &Wtf8) -> usize {
+    // For correctness, `CHUNK_SIZE` must be:
+    //
+    // - Less than or equal to 255, otherwise we'll overflow bytes in `counts`.
+    // - A multiple of `UNROLL_INNER`, otherwise our `break` inside the
+    //   `body.chunks(CHUNK_SIZE)` loop is incorrect.
+    //
+    // For performance, `CHUNK_SIZE` should be:
+    // - Relatively cheap to `/` against (so some simple sum of powers of two).
+    // - Large enough to avoid paying for the cost of the `sum_bytes_in_usize`
+    //   too often.
+    const CHUNK_SIZE: usize = 192;
+
+    // Check the properties of `CHUNK_SIZE` and `UNROLL_INNER` that are required
+    // for correctness.
+    const _: () = assert!(CHUNK_SIZE < 256);
+    const _: () = assert!(CHUNK_SIZE % UNROLL_INNER == 0);
+
+    // SAFETY: transmuting `[u8]` to `[usize]` is safe except for size
+    // differences which are handled by `align_to`.
+    let (head, body, tail) = unsafe { s.as_bytes().align_to::<usize>() };
+
+    // This should be quite rare, and basically exists to handle the degenerate
+    // cases where align_to fails (as well as miri under symbolic alignment
+    // mode).
+    //
+    // The `unlikely` helps discourage LLVM from inlining the body, which is
+    // nice, as we would rather not mark the `char_count_general_case` function
+    // as cold.
+    if unlikely(body.is_empty() || head.len() > USIZE_SIZE || tail.len() > USIZE_SIZE) {
+        return char_count_general_case(s.as_bytes());
+    }
+
+    let mut total = char_count_general_case(head) + char_count_general_case(tail);
+    // Split `body` into `CHUNK_SIZE` chunks to reduce the frequency with which
+    // we call `sum_bytes_in_usize`.
+    for chunk in body.chunks(CHUNK_SIZE) {
+        // We accumulate intermediate sums in `counts`, where each byte contains
+        // a subset of the sum of this chunk, like a `[u8; size_of::<usize>()]`.
+        let mut counts = 0;
+
+        let (unrolled_chunks, remainder) = slice_as_chunks::<_, UNROLL_INNER>(chunk);
+        for unrolled in unrolled_chunks {
+            for &word in unrolled {
+                // Because `CHUNK_SIZE` is < 256, this addition can't cause the
+                // count in any of the bytes to overflow into a subsequent byte.
+                counts += contains_non_continuation_byte(word);
+            }
+        }
+
+        // Sum the values in `counts` (which, again, is conceptually a `[u8;
+        // size_of::<usize>()]`), and accumulate the result into `total`.
+        total += sum_bytes_in_usize(counts);
+
+        // If there's any data in `remainder`, then handle it. This will only
+        // happen for the last `chunk` in `body.chunks()` (because `CHUNK_SIZE`
+        // is divisible by `UNROLL_INNER`), so we explicitly break at the end
+        // (which seems to help LLVM out).
+        if !remainder.is_empty() {
+            // Accumulate all the data in the remainder.
+            let mut counts = 0;
+            for &word in remainder {
+                counts += contains_non_continuation_byte(word);
+            }
+            total += sum_bytes_in_usize(counts);
+            break;
+        }
+    }
+    total
+}
+
+// Checks each byte of `w` to see if it contains the first byte in a UTF-8
+// sequence. Bytes in `w` which are continuation bytes are left as `0x00` (e.g.
+// false), and bytes which are non-continuation bytes are left as `0x01` (e.g.
+// true)
+#[inline]
+fn contains_non_continuation_byte(w: usize) -> usize {
+    const LSB: usize = usize_repeat_u8(0x01);
+    ((!w >> 7) | (w >> 6)) & LSB
+}
+
+// Morally equivalent to `values.to_ne_bytes().into_iter().sum::<usize>()`, but
+// more efficient.
+#[inline]
+fn sum_bytes_in_usize(values: usize) -> usize {
+    const LSB_SHORTS: usize = usize_repeat_u16(0x0001);
+    const SKIP_BYTES: usize = usize_repeat_u16(0x00ff);
+
+    let pair_sum: usize = (values & SKIP_BYTES) + ((values >> 8) & SKIP_BYTES);
+    pair_sum.wrapping_mul(LSB_SHORTS) >> ((USIZE_SIZE - 2) * 8)
+}
+
+// This is the most direct implementation of the concept of "count the number of
+// bytes in the string which are not continuation bytes", and is used for the
+// head and tail of the input string (the first and last item in the tuple
+// returned by `slice::align_to`).
+fn char_count_general_case(s: &[u8]) -> usize {
+    s.iter()
+        .filter(|&&byte| !super::core_str::utf8_is_cont_byte(byte))
+        .count()
+}
+
+// polyfills of unstable library features
+
+const fn usize_repeat_u8(x: u8) -> usize {
+    usize::from_ne_bytes([x; size_of::<usize>()])
+}
+
+const fn usize_repeat_u16(x: u16) -> usize {
+    let mut r = 0usize;
+    let mut i = 0;
+    while i < size_of::<usize>() {
+        // Use `wrapping_shl` to make it work on targets with 16-bit `usize`
+        r = r.wrapping_shl(16) | (x as usize);
+        i += 2;
+    }
+    r
+}
+
+fn slice_as_chunks<T, const N: usize>(slice: &[T]) -> (&[[T; N]], &[T]) {
+    assert!(N != 0, "chunk size must be non-zero");
+    let len_rounded_down = slice.len() / N * N;
+    // SAFETY: The rounded-down value is always the same or smaller than the
+    // original length, and thus must be in-bounds of the slice.
+    let (multiple_of_n, remainder) = unsafe { slice.split_at_unchecked(len_rounded_down) };
+    // SAFETY: We already panicked for zero, and ensured by construction
+    // that the length of the subslice is a multiple of N.
+    let array_slice = unsafe { slice_as_chunks_unchecked(multiple_of_n) };
+    (array_slice, remainder)
+}
+
+unsafe fn slice_as_chunks_unchecked<T, const N: usize>(slice: &[T]) -> &[[T; N]] {
+    let new_len = slice.len() / N;
+    // SAFETY: We cast a slice of `new_len * N` elements into
+    // a slice of `new_len` many `N` elements chunks.
+    unsafe { std::slice::from_raw_parts(slice.as_ptr().cast(), new_len) }
+}
+
+fn unlikely(x: bool) -> bool {
+    x
+}

stdlib/src/csv.rs

@@ -198,9 +198,9 @@ mod _csv {
     ) -> PyResult<Terminator> {
         match_class!(match obj.get_attr("lineterminator", vm)? {
             s @ PyStr => {
-                Ok(if s.as_str().as_bytes().eq(b"\r\n") {
+                Ok(if s.as_bytes().eq(b"\r\n") {
                     csv_core::Terminator::CRLF
-                } else if let Some(t) = s.as_str().as_bytes().first() {
+                } else if let Some(t) = s.as_bytes().first() {
                     // Due to limitations in the current implementation within csv_core
                     // the support for multiple characters in lineterminator is not complete.
                     // only capture the first character
@@ -942,7 +942,7 @@ mod _csv {
             ),
                 )
             })?;
-            let input = string.as_str().as_bytes();
+            let input = string.as_bytes();
             if input.is_empty() || input.starts_with(b"\n") {
                 return Ok(PyIterReturn::Return(vm.ctx.new_list(vec![]).into()));
             }
@@ -1101,11 +1101,11 @@ mod _csv {
                 let field: PyObjectRef = field?;
                 let stringified;
                 let data: &[u8] = match_class!(match field {
-                    ref s @ PyStr => s.as_str().as_bytes(),
+                    ref s @ PyStr => s.as_bytes(),
                     crate::builtins::PyNone => b"",
                     ref obj => {
                         stringified = obj.str(vm)?;
-                        stringified.as_str().as_bytes()
+                        stringified.as_bytes()
                     }
                 });
                 let mut input_offset = 0;

stdlib/src/pyexpat.rs

@@ -136,7 +136,7 @@ mod _pyexpat {
 
         #[pymethod(name = "Parse")]
         fn parse(&self, data: PyStrRef, _isfinal: OptionalArg<bool>, vm: &VirtualMachine) {
-            let reader = Cursor::<Vec<u8>>::new(data.as_str().as_bytes().to_vec());
+            let reader = Cursor::<Vec<u8>>::new(data.as_bytes().to_vec());
             let parser = self.create_config().create_reader(reader);
             self.do_parse(vm, parser);
         }

stdlib/src/pystruct.rs

@@ -27,30 +27,27 @@ pub(crate) mod _struct {
         fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
             // CPython turns str to bytes but we do reversed way here
             // The only performance difference is this transition cost
-            let fmt = match_class! {
-                match obj {
-                    s @ PyStr => if s.is_ascii() {
-                        Some(s)
-                    } else {
-                        None
-                    },
-                    b @ PyBytes => if b.is_ascii() {
-                        Some(unsafe {
-                            PyStr::new_ascii_unchecked(b.as_bytes().to_vec())
-                        }.into_ref(&vm.ctx))
-                    } else {
-                        None
-                    },
-                    other => return Err(vm.new_type_error(format!("Struct() argument 1 must be a str or bytes object, not {}", other.class().name()))),
-                }
-            }.ok_or_else(|| vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned()))?;
+            let fmt = match_class!(match obj {
+                s @ PyStr => s.is_ascii().then_some(s),
+                b @ PyBytes => ascii::AsciiStr::from_ascii(&b)
+                    .ok()
+                    .map(|s| vm.ctx.new_str(s)),
+                other =>
+                    return Err(vm.new_type_error(format!(
+                        "Struct() argument 1 must be a str or bytes object, not {}",
+                        other.class().name()
+                    ))),
+            })
+            .ok_or_else(|| {
+                vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned())
+            })?;
             Ok(IntoStructFormatBytes(fmt))
         }
     }
 
     impl IntoStructFormatBytes {
         fn format_spec(&self, vm: &VirtualMachine) -> PyResult<FormatSpec> {
-            FormatSpec::parse(self.0.as_str().as_bytes(), vm)
+            FormatSpec::parse(self.0.as_bytes(), vm)
         }
     }
 

stdlib/src/re.rs

@@ -9,10 +9,11 @@ mod re {
      * system.
      */
     use crate::vm::{
+        PyObjectRef, PyPayload, PyResult, VirtualMachine,
         builtins::{PyInt, PyIntRef, PyStr, PyStrRef},
         convert::{ToPyObject, TryFromObject},
         function::{OptionalArg, PosArgs},
-        match_class, PyObjectRef, PyResult, PyPayload, VirtualMachine,
+        match_class,
     };
     use num_traits::Signed;
     use regex::bytes::{Captures, Regex, RegexBuilder};
@@ -158,11 +159,9 @@ mod re {
     }
 
     fn do_sub(pattern: &PyPattern, repl: PyStrRef, search_text: PyStrRef, limit: usize) -> String {
-        let out = pattern.regex.replacen(
-            search_text.as_str().as_bytes(),
-            limit,
-            repl.as_str().as_bytes(),
-        );
+        let out = pattern
+            .regex
+            .replacen(search_text.as_bytes(), limit, repl.as_bytes());
         String::from_utf8_lossy(&out).into_owned()
     }
 
@@ -172,21 +171,21 @@ mod re {
         regex_text.push_str(pattern.regex.as_str());
         let regex = Regex::new(&regex_text).unwrap();
         regex
-            .captures(search_text.as_str().as_bytes())
+            .captures(search_text.as_bytes())
             .map(|captures| create_match(search_text.clone(), captures))
     }
 
     fn do_search(regex: &PyPattern, search_text: PyStrRef) -> Option<PyMatch> {
         regex
             .regex
-            .captures(search_text.as_str().as_bytes())
+            .captures(search_text.as_bytes())
             .map(|captures| create_match(search_text.clone(), captures))
     }
 
     fn do_findall(vm: &VirtualMachine, pattern: &PyPattern, search_text: PyStrRef) -> PyResult {
         let out = pattern
             .regex
-            .captures_iter(search_text.as_str().as_bytes())
+            .captures_iter(search_text.as_bytes())
             .map(|captures| match captures.len() {
                 1 => {
                     let full = captures.get(0).unwrap().as_bytes();
@@ -232,7 +231,7 @@ mod re {
             .map(|i| i.try_to_primitive::<usize>(vm))
             .transpose()?
             .unwrap_or(0);
-        let text = search_text.as_str().as_bytes();
+        let text = search_text.as_bytes();
         // essentially Regex::split, but it outputs captures as well
         let mut output = Vec::new();
         let mut last = 0;
@@ -332,9 +331,7 @@ mod re {
 
         #[pymethod]
         fn sub(&self, repl: PyStrRef, text: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> {
-            let replaced_text = self
-                .regex
-                .replace_all(text.as_str().as_bytes(), repl.as_str().as_bytes());
+            let replaced_text = self.regex.replace_all(text.as_bytes(), repl.as_bytes());
             let replaced_text = String::from_utf8_lossy(&replaced_text).into_owned();
             Ok(vm.ctx.new_str(replaced_text))
         }

stdlib/src/socket.rs

@@ -930,10 +930,15 @@ mod _socket {
             match family {
                 #[cfg(unix)]
                 c::AF_UNIX => {
+                    use crate::vm::function::ArgStrOrBytesLike;
                     use std::os::unix::ffi::OsStrExt;
-                    let buf = crate::vm::function::ArgStrOrBytesLike::try_from_object(vm, addr)?;
-                    let path = &*buf.borrow_bytes();
-                    socket2::SockAddr::unix(ffi::OsStr::from_bytes(path))
+                    let buf = ArgStrOrBytesLike::try_from_object(vm, addr)?;
+                    let bytes = &*buf.borrow_bytes();
+                    let path = match &buf {
+                        ArgStrOrBytesLike::Buf(_) => ffi::OsStr::from_bytes(bytes).into(),
+                        ArgStrOrBytesLike::Str(s) => vm.fsencode(s)?,
+                    };
+                    socket2::SockAddr::unix(path)
                         .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into())
                 }
                 c::AF_INET => {
@@ -1704,7 +1709,7 @@ mod _socket {
             let path = ffi::OsStr::as_bytes(addr.as_pathname().unwrap_or("".as_ref()).as_ref());
             let nul_pos = memchr::memchr(b'\0', path).unwrap_or(path.len());
             let path = ffi::OsStr::from_bytes(&path[..nul_pos]);
-            return vm.ctx.new_str(path.to_string_lossy()).into();
+            return vm.fsdecode(path).into();
         }
         // TODO: support more address families
         (String::new(), 0).to_pyobject(vm)

stdlib/src/sqlite.rs

@@ -2929,9 +2929,12 @@ mod _sqlite {
     }
 
     fn str_to_ptr_len(s: &PyStr, vm: &VirtualMachine) -> PyResult<(*const libc::c_char, i32)> {
-        let len = c_int::try_from(s.byte_len())
+        let s = s
+            .to_str()
+            .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed".to_owned()))?;
+        let len = c_int::try_from(s.len())
             .map_err(|_| vm.new_overflow_error("TEXT longer than INT_MAX bytes".to_owned()))?;
-        let ptr = s.as_str().as_ptr().cast();
+        let ptr = s.as_ptr().cast();
         Ok((ptr, len))
     }
 
@@ -3000,7 +3003,7 @@ mod _sqlite {
     ) -> PyResult<*const libc::c_char> {
         BEGIN_STATEMENTS
             .iter()
-            .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_str().as_bytes()))
+            .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_bytes()))
             .map(|&x| x.as_ptr().cast())
             .ok_or_else(|| {
                 vm.new_value_error(

stdlib/src/ssl.rs

@@ -403,8 +403,8 @@ mod _ssl {
             .to_str()
             .unwrap();
         let (cert_file, cert_dir) = get_cert_file_dir();
-        let cert_file = OsPath::new_str(cert_file).filename(vm)?;
-        let cert_dir = OsPath::new_str(cert_dir).filename(vm)?;
+        let cert_file = OsPath::new_str(cert_file).filename(vm);
+        let cert_dir = OsPath::new_str(cert_dir).filename(vm);
         Ok((cert_file_env, cert_file, cert_dir_env, cert_dir))
     }
 
@@ -708,7 +708,7 @@ mod _ssl {
                         if !s.is_ascii() {
                             return Err(invalid_cadata(vm));
                         }
-                        X509::stack_from_pem(s.as_str().as_bytes())
+                        X509::stack_from_pem(s.as_bytes())
                     }
                     Either::B(b) => b.with_ref(x509_stack_from_der),
                 };

stdlib/src/unicodedata.rs

@@ -65,6 +65,7 @@ mod unicodedata {
         function::OptionalArg,
     };
     use itertools::Itertools;
+    use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
     use ucd::{Codepoint, EastAsianWidth};
     use unic_char_property::EnumeratedCharProperty;
     use unic_normal::StrNormalForm;
@@ -84,14 +85,23 @@ mod unicodedata {
             Self { unic_version }
         }
 
-        fn check_age(&self, c: char) -> bool {
-            Age::of(c).is_some_and(|age| age.actual() <= self.unic_version)
+        fn check_age(&self, c: CodePoint) -> bool {
+            c.to_char()
+                .is_none_or(|c| Age::of(c).is_some_and(|age| age.actual() <= self.unic_version))
         }
 
-        fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<Option<char>> {
-            let c = character.as_str().chars().exactly_one().map_err(|_| {
-                vm.new_type_error("argument must be an unicode character, not str".to_owned())
-            })?;
+        fn extract_char(
+            &self,
+            character: PyStrRef,
+            vm: &VirtualMachine,
+        ) -> PyResult<Option<CodePoint>> {
+            let c = character
+                .as_wtf8()
+                .code_points()
+                .exactly_one()
+                .map_err(|_| {
+                    vm.new_type_error("argument must be an unicode character, not str".to_owned())
+                })?;
 
             Ok(self.check_age(c).then_some(c))
         }
@@ -103,7 +113,10 @@ mod unicodedata {
         fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
             Ok(self
                 .extract_char(character, vm)?
-                .map_or(GeneralCategory::Unassigned, GeneralCategory::of)
+                .map_or(GeneralCategory::Unassigned, |c| {
+                    c.to_char()
+                        .map_or(GeneralCategory::Surrogate, GeneralCategory::of)
+                })
                 .abbr_name()
                 .to_owned())
         }
@@ -111,7 +124,7 @@ mod unicodedata {
         #[pymethod]
         fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
             if let Some(character) = unicode_names2::character(name.as_str()) {
-                if self.check_age(character) {
+                if self.check_age(character.into()) {
                     return Ok(character.to_string());
                 }
             }
@@ -129,7 +142,7 @@ mod unicodedata {
 
             if let Some(c) = c {
                 if self.check_age(c) {
-                    if let Some(name) = unicode_names2::name(c) {
+                    if let Some(name) = c.to_char().and_then(unicode_names2::name) {
                         return Ok(vm.ctx.new_str(name.to_string()).into());
                     }
                 }
@@ -144,7 +157,10 @@ mod unicodedata {
             vm: &VirtualMachine,
         ) -> PyResult<&'static str> {
             let bidi = match self.extract_char(character, vm)? {
-                Some(c) => BidiClass::of(c).abbr_name(),
+                Some(c) => c
+                    .to_char()
+                    .map_or(BidiClass::LeftToRight, BidiClass::of)
+                    .abbr_name(),
                 None => "",
             };
             Ok(bidi)
@@ -159,19 +175,20 @@ mod unicodedata {
         ) -> PyResult<&'static str> {
             Ok(self
                 .extract_char(character, vm)?
+                .and_then(|c| c.to_char())
                 .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width())
                 .abbr_name())
         }
 
         #[pymethod]
-        fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<String> {
+        fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<Wtf8Buf> {
             use super::NormalizeForm::*;
-            let text = unistr.as_str();
+            let text = unistr.as_wtf8();
             let normalized_text = match form {
-                Nfc => text.nfc().collect::<String>(),
-                Nfkc => text.nfkc().collect::<String>(),
-                Nfd => text.nfd().collect::<String>(),
-                Nfkd => text.nfkd().collect::<String>(),
+                Nfc => text.map_utf8(|s| s.nfc()).collect(),
+                Nfkc => text.map_utf8(|s| s.nfkc()).collect(),
+                Nfd => text.map_utf8(|s| s.nfd()).collect(),
+                Nfkd => text.map_utf8(|s| s.nfkd()).collect(),
             };
             Ok(normalized_text)
         }

vm/src/anystr.rs

@@ -7,28 +7,13 @@ use crate::{
 use num_traits::{cast::ToPrimitive, sign::Signed};
 
 #[derive(FromArgs)]
-pub struct SplitArgs<T: TryFromObject + AnyStrWrapper> {
+pub struct SplitArgs<T: TryFromObject> {
     #[pyarg(any, default)]
     sep: Option<T>,
     #[pyarg(any, default = "-1")]
     maxsplit: isize,
 }
 
-impl<T: TryFromObject + AnyStrWrapper> SplitArgs<T> {
-    pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option<T>, isize)> {
-        let sep = if let Some(s) = self.sep {
-            let sep = s.as_ref();
-            if sep.is_empty() {
-                return Err(vm.new_value_error("empty separator".to_owned()));
-            }
-            Some(s)
-        } else {
-            None
-        };
-        Ok((sep, self.maxsplit))
-    }
-}
-
 #[derive(FromArgs)]
 pub struct SplitLinesArgs {
     #[pyarg(any, default = "false")]
@@ -132,9 +117,9 @@ impl StringRange for std::ops::Range<usize> {
     }
 }
 
-pub trait AnyStrWrapper {
-    type Str: ?Sized + AnyStr;
-    fn as_ref(&self) -> &Self::Str;
+pub trait AnyStrWrapper<S: AnyStr + ?Sized> {
+    fn as_ref(&self) -> Option<&S>;
+    fn is_empty(&self) -> bool;
 }
 
 pub trait AnyStrContainer<S>
@@ -146,15 +131,18 @@ where
     fn push_str(&mut self, s: &S);
 }
 
-pub trait AnyStr {
-    type Char: Copy;
-    type Container: AnyStrContainer<Self> + Extend<Self::Char>;
+pub trait AnyChar: Copy {
+    fn is_lowercase(self) -> bool;
+    fn is_uppercase(self) -> bool;
+    fn bytes_len(self) -> usize;
+}
 
-    fn element_bytes_len(c: Self::Char) -> usize;
+pub trait AnyStr {
+    type Char: AnyChar;
+    type Container: AnyStrContainer<Self> + Extend<Self::Char>;
 
     fn to_container(&self) -> Self::Container;
     fn as_bytes(&self) -> &[u8];
-    fn chars(&self) -> impl Iterator<Item = char>;
     fn elements(&self) -> impl Iterator<Item = Self::Char>;
     fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self;
     // FIXME: get_chars is expensive for str
@@ -172,29 +160,35 @@ pub trait AnyStr {
         new
     }
 
-    fn py_split<T, SP, SN, SW, R>(
+    fn py_split<T, SP, SN, SW>(
         &self,
         args: SplitArgs<T>,
         vm: &VirtualMachine,
+        full_obj: impl FnOnce() -> PyObjectRef,
         split: SP,
         splitn: SN,
         splitw: SW,
-    ) -> PyResult<Vec<R>>
+    ) -> PyResult<Vec<PyObjectRef>>
     where
-        T: TryFromObject + AnyStrWrapper<Str = Self>,
-        SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<R>,
-        SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<R>,
-        SW: Fn(&Self, isize, &VirtualMachine) -> Vec<R>,
+        T: TryFromObject + AnyStrWrapper<Self>,
+        SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<PyObjectRef>,
+        SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<PyObjectRef>,
+        SW: Fn(&Self, isize, &VirtualMachine) -> Vec<PyObjectRef>,
     {
-        let (sep, maxsplit) = args.get_value(vm)?;
-        let splits = if let Some(pattern) = sep {
-            if maxsplit < 0 {
-                split(self, pattern.as_ref(), vm)
+        if args.sep.as_ref().is_some_and(|sep| sep.is_empty()) {
+            return Err(vm.new_value_error("empty separator".to_owned()));
+        }
+        let splits = if let Some(pattern) = args.sep {
+            let Some(pattern) = pattern.as_ref() else {
+                return Ok(vec![full_obj()]);
+            };
+            if args.maxsplit < 0 {
+                split(self, pattern, vm)
             } else {
-                splitn(self, pattern.as_ref(), (maxsplit + 1) as usize, vm)
+                splitn(self, pattern, (args.maxsplit + 1) as usize, vm)
             }
         } else {
-            splitw(self, maxsplit, vm)
+            splitw(self, args.maxsplit, vm)
         };
         Ok(splits)
     }
@@ -242,13 +236,19 @@ pub trait AnyStr {
         func_default: FD,
     ) -> &'a Self
     where
-        S: AnyStrWrapper<Str = Self>,
+        S: AnyStrWrapper<Self>,
         FC: Fn(&'a Self, &Self) -> &'a Self,
         FD: Fn(&'a Self) -> &'a Self,
     {
         let chars = chars.flatten();
         match chars {
-            Some(chars) => func_chars(self, chars.as_ref()),
+            Some(chars) => {
+                if let Some(chars) = chars.as_ref() {
+                    func_chars(self, chars)
+                } else {
+                    self
+                }
+            }
             None => func_default(self),
         }
     }
@@ -281,7 +281,7 @@ pub trait AnyStr {
 
     fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container {
         let mut u = Self::Container::with_capacity(
-            (left + right) * Self::element_bytes_len(fillchar) + self.bytes_len(),
+            (left + right) * fillchar.bytes_len() + self.bytes_len(),
         );
         u.extend(std::iter::repeat(fillchar).take(left));
         u.push_str(self);
@@ -305,19 +305,17 @@ pub trait AnyStr {
 
     fn py_join(
         &self,
-        mut iter: impl std::iter::Iterator<
-            Item = PyResult<impl AnyStrWrapper<Str = Self> + TryFromObject>,
-        >,
+        mut iter: impl std::iter::Iterator<Item = PyResult<impl AnyStrWrapper<Self> + TryFromObject>>,
     ) -> PyResult<Self::Container> {
         let mut joined = if let Some(elem) = iter.next() {
-            elem?.as_ref().to_container()
+            elem?.as_ref().unwrap().to_container()
         } else {
             return Ok(Self::Container::new());
         };
         for elem in iter {
             let elem = elem?;
             joined.push_str(self);
-            joined.push_str(elem.as_ref());
+            joined.push_str(elem.as_ref().unwrap());
         }
         Ok(joined)
     }
@@ -403,25 +401,34 @@ pub trait AnyStr {
         rustpython_common::str::zfill(self.as_bytes(), width)
     }
 
-    fn py_iscase<F, G>(&self, is_case: F, is_opposite: G) -> bool
-    where
-        F: Fn(char) -> bool,
-        G: Fn(char) -> bool,
-    {
-        // Unified form of CPython functions:
-        //  _Py_bytes_islower
-        //   Py_bytes_isupper
-        //  unicode_islower_impl
-        //  unicode_isupper_impl
-        let mut cased = false;
-        for c in self.chars() {
-            if is_opposite(c) {
+    // Unified form of CPython functions:
+    //  _Py_bytes_islower
+    //  unicode_islower_impl
+    fn py_islower(&self) -> bool {
+        let mut lower = false;
+        for c in self.elements() {
+            if c.is_uppercase() {
                 return false;
-            } else if !cased && is_case(c) {
-                cased = true
+            } else if !lower && c.is_lowercase() {
+                lower = true
             }
         }
-        cased
+        lower
+    }
+
+    // Unified form of CPython functions:
+    //   Py_bytes_isupper
+    //  unicode_isupper_impl
+    fn py_isupper(&self) -> bool {
+        let mut upper = false;
+        for c in self.elements() {
+            if c.is_lowercase() {
+                return false;
+            } else if !upper && c.is_uppercase() {
+                upper = true
+            }
+        }
+        upper
     }
 }
 

vm/src/builtins/genericalias.rs

@@ -124,7 +124,7 @@ impl PyGenericAlias {
         Ok(format!(
             "{}[{}]",
             repr_item(self.origin.clone().into(), vm)?,
-            if self.args.len() == 0 {
+            if self.args.is_empty() {
                 "()".to_owned()
             } else {
                 self.args
@@ -261,23 +261,20 @@ fn subs_tvars(
         .and_then(|sub_params| {
             PyTupleRef::try_from_object(vm, sub_params)
                 .ok()
-                .and_then(|sub_params| {
-                    if sub_params.len() > 0 {
-                        let sub_args = sub_params
-                            .iter()
-                            .map(|arg| {
-                                if let Some(idx) = tuple_index(params, arg) {
-                                    argitems[idx].clone()
-                                } else {
-                                    arg.clone()
-                                }
-                            })
-                            .collect::<Vec<_>>();
-                        let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into();
-                        Some(obj.get_item(&*sub_args, vm))
-                    } else {
-                        None
-                    }
+                .filter(|sub_params| !sub_params.is_empty())
+                .map(|sub_params| {
+                    let sub_args = sub_params
+                        .iter()
+                        .map(|arg| {
+                            if let Some(idx) = tuple_index(params, arg) {
+                                argitems[idx].clone()
+                            } else {
+                                arg.clone()
+                            }
+                        })
+                        .collect::<Vec<_>>();
+                    let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into();
+                    obj.get_item(&*sub_args, vm)
                 })
         })
         .unwrap_or(Ok(obj))

vm/src/builtins/object.rs

@@ -159,7 +159,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine)
                 slots.set_item(name.as_str(), value, vm).unwrap();
             }
 
-            if slots.len() > 0 {
+            if !slots.is_empty() {
                 return (state, slots).to_pyresult(vm);
             }
         }

vm/src/builtins/set.rs

@@ -784,9 +784,7 @@ impl Initializer for PySet {
     type Args = OptionalArg<PyObjectRef>;
 
     fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> {
-        if zelf.len() > 0 {
-            zelf.clear();
-        }
+        zelf.clear();
         if let OptionalArg::Present(it) = iterable {
             zelf.update(PosArgs::new(vec![it]), vm)?;
         }

vm/src/builtins/type.rs

@@ -768,7 +768,7 @@ impl PyType {
                 value.class().slot_name(),
             ))
         })?;
-        if name.as_str().as_bytes().contains(&0) {
+        if name.as_bytes().contains(&0) {
             return Err(vm.new_value_error("type name must not contain null characters".to_owned()));
         }
 
@@ -811,7 +811,7 @@ impl Constructor for PyType {
         let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyDictRef, KwArgs) =
             args.clone().bind(vm)?;
 
-        if name.as_str().as_bytes().contains(&0) {
+        if name.as_bytes().contains(&0) {
             return Err(vm.new_value_error("type name must not contain null characters".to_owned()));
         }
 

vm/src/builtins/union.rs

@@ -204,7 +204,7 @@ impl PyUnion {
             vm,
         )?;
         let mut res;
-        if new_args.len() == 0 {
+        if new_args.is_empty() {
             res = make_union(&new_args, vm);
         } else {
             res = new_args.fast_getitem(0);

vm/src/bytesinner.rs

@@ -355,13 +355,11 @@ impl PyBytesInner {
     }
 
     pub fn islower(&self) -> bool {
-        self.elements
-            .py_iscase(char::is_lowercase, char::is_uppercase)
+        self.elements.py_islower()
     }
 
     pub fn isupper(&self) -> bool {
-        self.elements
-            .py_iscase(char::is_uppercase, char::is_lowercase)
+        self.elements.py_isupper()
     }
 
     pub fn isspace(&self) -> bool {
@@ -654,6 +652,7 @@ impl PyBytesInner {
         let elements = self.elements.py_split(
             options,
             vm,
+            || convert(&self.elements, vm),
             |v, s, vm| v.split_str(s).map(|v| convert(v, vm)).collect(),
             |v, s, n, vm| v.splitn_str(n, s).map(|v| convert(v, vm)).collect(),
             |v, n, vm| v.py_split_whitespace(n, |v| convert(v, vm)),
@@ -673,6 +672,7 @@ impl PyBytesInner {
         let mut elements = self.elements.py_split(
             options,
             vm,
+            || convert(&self.elements, vm),
             |v, s, vm| v.rsplit_str(s).map(|v| convert(v, vm)).collect(),
             |v, s, n, vm| v.rsplitn_str(n, s).map(|v| convert(v, vm)).collect(),
             |v, n, vm| v.py_rsplit_whitespace(n, |v| convert(v, vm)),
@@ -998,10 +998,12 @@ pub trait ByteOr: ToPrimitive {
 
 impl ByteOr for BigInt {}
 
-impl AnyStrWrapper for PyBytesInner {
-    type Str = [u8];
-    fn as_ref(&self) -> &[u8] {
-        &self.elements
+impl AnyStrWrapper<[u8]> for PyBytesInner {
+    fn as_ref(&self) -> Option<&[u8]> {
+        Some(&self.elements)
+    }
+    fn is_empty(&self) -> bool {
+        self.elements.is_empty()
     }
 }
 
@@ -1021,14 +1023,22 @@ impl AnyStrContainer<[u8]> for Vec<u8> {
 
 const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b];
 
+impl anystr::AnyChar for u8 {
+    fn is_lowercase(self) -> bool {
+        self.is_ascii_lowercase()
+    }
+    fn is_uppercase(self) -> bool {
+        self.is_ascii_uppercase()
+    }
+    fn bytes_len(self) -> usize {
+        1
+    }
+}
+
 impl AnyStr for [u8] {
     type Char = u8;
     type Container = Vec<u8>;
 
-    fn element_bytes_len(_: u8) -> usize {
-        1
-    }
-
     fn to_container(&self) -> Self::Container {
         self.to_vec()
     }
@@ -1037,10 +1047,6 @@ impl AnyStr for [u8] {
         self
     }
 
-    fn chars(&self) -> impl Iterator<Item = char> {
-        bstr::ByteSlice::chars(self)
-    }
-
     fn elements(&self) -> impl Iterator<Item = u8> {
         self.iter().copied()
     }

vm/src/cformat.rs

@@ -2,6 +2,7 @@
 //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting).
 
 use crate::common::cformat::*;
+use crate::common::wtf8::{CodePoint, Wtf8, Wtf8Buf};
 use crate::{
     AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine,
     builtins::{
@@ -125,13 +126,13 @@ fn spec_format_string(
     spec: &CFormatSpec,
     obj: PyObjectRef,
     idx: usize,
-) -> PyResult<String> {
+) -> PyResult<Wtf8Buf> {
     match &spec.format_type {
         CFormatType::String(conversion) => {
             let result = match conversion {
                 CFormatConversion::Ascii => builtins::ascii(obj, vm)?.into(),
-                CFormatConversion::Str => obj.str(vm)?.as_str().to_owned(),
-                CFormatConversion::Repr => obj.repr(vm)?.as_str().to_owned(),
+                CFormatConversion::Str => obj.str(vm)?.as_wtf8().to_owned(),
+                CFormatConversion::Repr => obj.repr(vm)?.as_wtf8().to_owned(),
                 CFormatConversion::Bytes => {
                     // idx is the position of the %, we want the position of the b
                     return Err(vm.new_value_error(format!(
@@ -146,16 +147,18 @@ fn spec_format_string(
             CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => {
                 match_class!(match &obj {
                     ref i @ PyInt => {
-                        Ok(spec.format_number(i.as_bigint()))
+                        Ok(spec.format_number(i.as_bigint()).into())
                     }
                     ref f @ PyFloat => {
-                        Ok(spec.format_number(&try_f64_to_bigint(f.to_f64(), vm)?))
+                        Ok(spec
+                            .format_number(&try_f64_to_bigint(f.to_f64(), vm)?)
+                            .into())
                     }
                     obj => {
                         if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) {
                             let result = method?.call((), vm)?;
                             if let Some(i) = result.payload::<PyInt>() {
-                                return Ok(spec.format_number(i.as_bigint()));
+                                return Ok(spec.format_number(i.as_bigint()).into());
                             }
                         }
                         Err(vm.new_type_error(format!(
@@ -168,7 +171,7 @@ fn spec_format_string(
             }
             _ => {
                 if let Some(i) = obj.payload::<PyInt>() {
-                    Ok(spec.format_number(i.as_bigint()))
+                    Ok(spec.format_number(i.as_bigint()).into())
                 } else {
                     Err(vm.new_type_error(format!(
                         "%{} format: an integer is required, not {}",
@@ -180,21 +183,21 @@ fn spec_format_string(
         },
         CFormatType::Float(_) => {
             let value = ArgIntoFloat::try_from_object(vm, obj)?;
-            Ok(spec.format_float(value.into()))
+            Ok(spec.format_float(value.into()).into())
         }
         CFormatType::Character(CCharacterType::Character) => {
             if let Some(i) = obj.payload::<PyInt>() {
                 let ch = i
                     .as_bigint()
                     .to_u32()
-                    .and_then(char::from_u32)
+                    .and_then(CodePoint::from_u32)
                     .ok_or_else(|| {
                         vm.new_overflow_error("%c arg not in range(0x110000)".to_owned())
                     })?;
                 return Ok(spec.format_char(ch));
             }
             if let Some(s) = obj.payload::<PyStr>() {
-                if let Ok(ch) = s.as_str().chars().exactly_one() {
+                if let Ok(ch) = s.as_wtf8().code_points().exactly_one() {
                     return Ok(spec.format_char(ch));
                 }
             }
@@ -374,17 +377,16 @@ pub(crate) fn cformat_bytes(
 
 pub(crate) fn cformat_string(
     vm: &VirtualMachine,
-    format_string: &str,
+    format_string: &Wtf8,
     values_obj: PyObjectRef,
-) -> PyResult<String> {
-    let format = format_string
-        .parse::<CFormatString>()
+) -> PyResult<Wtf8Buf> {
+    let format = CFormatWtf8::parse_from_wtf8(format_string)
         .map_err(|err| vm.new_value_error(err.to_string()))?;
     let (num_specifiers, mapping_required) = format
         .check_specifiers()
         .ok_or_else(|| specifier_error(vm))?;
 
-    let mut result = String::new();
+    let mut result = Wtf8Buf::new();
 
     let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__))
         && !values_obj.fast_isinstance(vm.ctx.types.tuple_type)
@@ -399,7 +401,7 @@ pub(crate) fn cformat_string(
         {
             for (_, part) in format.iter() {
                 match part {
-                    CFormatPart::Literal(literal) => result.push_str(literal),
+                    CFormatPart::Literal(literal) => result.push_wtf8(literal),
                     CFormatPart::Spec(_) => unreachable!(),
                 }
             }
@@ -415,11 +417,11 @@ pub(crate) fn cformat_string(
         return if is_mapping {
             for (idx, part) in format {
                 match part {
-                    CFormatPart::Literal(literal) => result.push_str(&literal),
+                    CFormatPart::Literal(literal) => result.push_wtf8(&literal),
                     CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => {
                         let value = values_obj.get_item(&mapping_key.unwrap(), vm)?;
                         let part_result = spec_format_string(vm, &spec, value, idx)?;
-                        result.push_str(&part_result);
+                        result.push_wtf8(&part_result);
                     }
                 }
             }
@@ -439,7 +441,7 @@ pub(crate) fn cformat_string(
 
     for (idx, part) in format {
         match part {
-            CFormatPart::Literal(literal) => result.push_str(&literal),
+            CFormatPart::Literal(literal) => result.push_wtf8(&literal),
             CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => {
                 try_update_quantity_from_tuple(
                     vm,
@@ -456,7 +458,7 @@ pub(crate) fn cformat_string(
                     }
                 }?;
                 let part_result = spec_format_string(vm, &spec, value, idx)?;
-                result.push_str(&part_result);
+                result.push_wtf8(&part_result);
             }
         }
     }

vm/src/codecs.rs

@@ -1,3 +1,5 @@
+use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
+
 use crate::{
     AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine,
     builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef},
@@ -424,12 +426,13 @@ fn xmlcharrefreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(
     }
     let range = extract_unicode_error_range(&err, vm)?;
     let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
-    let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
+    let s_after_start =
+        crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default();
     let num_chars = range.len();
     // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#;
     let mut out = String::with_capacity(num_chars * 6);
-    for c in s_after_start.chars().take(num_chars) {
-        write!(out, "&#{};", c as u32).unwrap()
+    for c in s_after_start.code_points().take(num_chars) {
+        write!(out, "&#{};", c.to_u32()).unwrap()
     }
     Ok((out, range.end))
 }
@@ -448,12 +451,13 @@ fn backslashreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(S
     }
     let range = extract_unicode_error_range(&err, vm)?;
     let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
-    let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
+    let s_after_start =
+        crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default();
     let num_chars = range.len();
     // minimum 4 output bytes per char: \xNN
     let mut out = String::with_capacity(num_chars * 4);
-    for c in s_after_start.chars().take(num_chars) {
-        let c = c as u32;
+    for c in s_after_start.code_points().take(num_chars) {
+        let c = c.to_u32();
         if c >= 0x10000 {
             write!(out, "\\U{c:08x}").unwrap();
         } else if c >= 0x100 {
@@ -470,12 +474,12 @@ fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String
         let range = extract_unicode_error_range(&err, vm)?;
         let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
         let s_after_start =
-            crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
+            crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default();
         let num_chars = range.len();
         let mut out = String::with_capacity(num_chars * 4);
-        for c in s_after_start.chars().take(num_chars) {
-            let c_u32 = c as u32;
-            if let Some(c_name) = unicode_names2::name(c) {
+        for c in s_after_start.code_points().take(num_chars) {
+            let c_u32 = c.to_u32();
+            if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) {
                 write!(out, "\\N{{{c_name}}}").unwrap();
             } else if c_u32 >= 0x10000 {
                 write!(out, "\\U{c_u32:08x}").unwrap();
@@ -570,10 +574,11 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb
             return Err(err.downcast().unwrap());
         }
         let s_after_start =
-            crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
+            crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default();
         let num_chars = range.len();
         let mut out: Vec<u8> = Vec::with_capacity(num_chars * 4);
-        for c in s_after_start.chars().take(num_chars).map(|x| x as u32) {
+        for c in s_after_start.code_points().take(num_chars) {
+            let c = c.to_u32();
             if !(0xd800..=0xdfff).contains(&c) {
                 // Not a surrogate, fail with original exception
                 return Err(err.downcast().unwrap());
@@ -671,7 +676,7 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb
         }
 
         Ok((
-            vm.new_pyobj(format!("\\x{c:x?}")),
+            vm.new_pyobj(CodePoint::from_u32(c).unwrap()),
             range.start + byte_length,
         ))
     } else {
@@ -683,11 +688,11 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py
     if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) {
         let range = extract_unicode_error_range(&err, vm)?;
         let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
-        let s_after_start =
-            crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or("");
+        let s_after_start = crate::common::str::try_get_codepoints(object.as_wtf8(), range.start..)
+            .unwrap_or_default();
         let mut out: Vec<u8> = Vec::with_capacity(range.len());
-        for ch in s_after_start.chars().take(range.len()) {
-            let ch = ch as u32;
+        for ch in s_after_start.code_points().take(range.len()) {
+            let ch = ch.to_u32();
             if !(0xdc80..=0xdcff).contains(&ch) {
                 // Not a UTF-8b surrogate, fail with original exception
                 return Err(err.downcast().unwrap());
@@ -702,14 +707,14 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py
         let object = PyBytesRef::try_from_object(vm, object)?;
         let p = &object.as_bytes()[range.clone()];
         let mut consumed = 0;
-        let mut replace = String::with_capacity(4 * range.len());
+        let mut replace = Wtf8Buf::with_capacity(4 * range.len());
         while consumed < 4 && consumed < range.len() {
-            let c = p[consumed] as u32;
+            let c = p[consumed] as u16;
             // Refuse to escape ASCII bytes
             if c < 128 {
                 break;
             }
-            write!(replace, "#{}", 0xdc00 + c).unwrap();
+            replace.push(CodePoint::from(0xdc00 + c));
             consumed += 1;
         }
         if consumed == 0 {

vm/src/dictdatatype.rs

@@ -12,6 +12,7 @@ use crate::{
     common::{
         hash,
         lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard},
+        wtf8::{Wtf8, Wtf8Buf},
     },
     object::{Traverse, TraverseFn},
 };
@@ -750,7 +751,7 @@ impl DictKey for Py<PyStr> {
         if self.is(other_key) {
             Ok(true)
         } else if let Some(pystr) = other_key.payload_if_exact::<PyStr>(vm) {
-            Ok(pystr.as_str() == self.as_str())
+            Ok(self.as_wtf8() == pystr.as_wtf8())
         } else {
             vm.bool_eq(self.as_object(), other_key)
         }
@@ -834,7 +835,7 @@ impl DictKey for str {
 
     fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> {
         if let Some(pystr) = other_key.payload_if_exact::<PyStr>(vm) {
-            Ok(pystr.as_str() == self)
+            Ok(pystr.as_wtf8() == self)
         } else {
             // Fall back to PyObjectRef implementation.
             let s = vm.ctx.new_str(self);
@@ -871,6 +872,63 @@ impl DictKey for String {
     }
 }
 
+impl DictKey for Wtf8 {
+    type Owned = Wtf8Buf;
+    #[inline(always)]
+    fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned {
+        self.to_owned()
+    }
+    #[inline]
+    fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> {
+        // follow a similar route as the hashing of PyStrRef
+        Ok(vm.state.hash_secret.hash_bytes(self.as_bytes()))
+    }
+    #[inline(always)]
+    fn key_is(&self, _other: &PyObject) -> bool {
+        // No matter who the other pyobject is, we are never the same thing, since
+        // we are a str, not a pyobject.
+        false
+    }
+
+    fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> {
+        if let Some(pystr) = other_key.payload_if_exact::<PyStr>(vm) {
+            Ok(pystr.as_wtf8() == self)
+        } else {
+            // Fall back to PyObjectRef implementation.
+            let s = vm.ctx.new_str(self);
+            s.key_eq(vm, other_key)
+        }
+    }
+
+    fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> {
+        Err(vm.new_type_error("'str' object cannot be interpreted as an integer".to_owned()))
+    }
+}
+
+impl DictKey for Wtf8Buf {
+    type Owned = Wtf8Buf;
+    #[inline]
+    fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned {
+        self.clone()
+    }
+
+    fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> {
+        (**self).key_hash(vm)
+    }
+
+    fn key_is(&self, other: &PyObject) -> bool {
+        (**self).key_is(other)
+    }
+
+    fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> {
+        (**self).key_eq(vm, other_key)
+    }
+
+    fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> {
+        (**self).key_as_isize(vm)
+    }
+}
+
 impl DictKey for [u8] {
     type Owned = Vec<u8>;
     #[inline(always)]

vm/src/format.rs

@@ -7,6 +7,7 @@ use crate::{
 };
 
 use crate::common::format::*;
+use crate::common::wtf8::{Wtf8, Wtf8Buf};
 
 impl IntoPyException for FormatSpecError {
     fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef {
@@ -62,18 +63,18 @@ fn format_internal(
     vm: &VirtualMachine,
     format: &FormatString,
     field_func: &mut impl FnMut(FieldType) -> PyResult,
-) -> PyResult<String> {
-    let mut final_string = String::new();
+) -> PyResult<Wtf8Buf> {
+    let mut final_string = Wtf8Buf::new();
     for part in &format.format_parts {
         let pystr;
-        let result_string: &str = match part {
+        let result_string: &Wtf8 = match part {
             FormatPart::Field {
                 field_name,
                 conversion_spec,
                 format_spec,
             } => {
                 let FieldName { field_type, parts } =
-                    FieldName::parse(field_name.as_str()).map_err(|e| e.to_pyexception(vm))?;
+                    FieldName::parse(field_name).map_err(|e| e.to_pyexception(vm))?;
 
                 let mut argument = field_func(field_type)?;
 
@@ -113,7 +114,7 @@ fn format_internal(
             }
             FormatPart::Literal(literal) => literal,
         };
-        final_string.push_str(result_string);
+        final_string.push_wtf8(result_string);
     }
     Ok(final_string)
 }
@@ -122,7 +123,7 @@ pub(crate) fn format(
     format: &FormatString,
     arguments: &FuncArgs,
     vm: &VirtualMachine,
-) -> PyResult<String> {
+) -> PyResult<Wtf8Buf> {
     let mut auto_argument_index: usize = 0;
     let mut seen_index = false;
     format_internal(vm, format, &mut |field_type| match field_type {
@@ -154,8 +155,10 @@ pub(crate) fn format(
                 .cloned()
                 .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned()))
         }
-        FieldType::Keyword(keyword) => arguments
-            .get_optional_kwarg(&keyword)
+        FieldType::Keyword(keyword) => keyword
+            .as_str()
+            .ok()
+            .and_then(|keyword| arguments.get_optional_kwarg(keyword))
             .ok_or_else(|| vm.new_key_error(vm.ctx.new_str(keyword).into())),
     })
 }
@@ -164,7 +167,7 @@ pub(crate) fn format_map(
     format: &FormatString,
     dict: &PyObject,
     vm: &VirtualMachine,
-) -> PyResult<String> {
+) -> PyResult<Wtf8Buf> {
     format_internal(vm, format, &mut |field_type| match field_type {
         FieldType::Auto | FieldType::Index(_) => {
             Err(vm.new_value_error("Format string contains positional fields".to_owned()))

vm/src/frame.rs

@@ -21,6 +21,7 @@ use crate::{
 };
 use indexmap::IndexMap;
 use itertools::Itertools;
+use rustpython_common::wtf8::Wtf8Buf;
 #[cfg(feature = "threading")]
 use std::sync::atomic;
 use std::{fmt, iter::zip};
@@ -697,8 +698,8 @@ impl ExecutingFrame<'_> {
                     .pop_multiple(size.get(arg) as usize)
                     .as_slice()
                     .iter()
-                    .map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref())
-                    .collect::<String>();
+                    .map(|pyobj| pyobj.payload::<PyStr>().unwrap())
+                    .collect::<Wtf8Buf>();
                 let str_obj = vm.ctx.new_str(s);
                 self.push_value(str_obj.into());
                 Ok(None)
@@ -1468,7 +1469,7 @@ impl ExecutingFrame<'_> {
         let kwarg_names = kwarg_names
             .as_slice()
             .iter()
-            .map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref().to_owned());
+            .map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_str().to_owned());
         FuncArgs::with_kwargs_names(args, kwarg_names)
     }
 

vm/src/function/buffer.rs

@@ -152,7 +152,7 @@ impl ArgStrOrBytesLike {
     pub fn borrow_bytes(&self) -> BorrowedValue<'_, [u8]> {
         match self {
             Self::Buf(b) => b.borrow_buf(),
-            Self::Str(s) => s.as_str().as_bytes().into(),
+            Self::Str(s) => s.as_bytes().into(),
         }
     }
 }
@@ -195,7 +195,7 @@ impl ArgAsciiBuffer {
     #[inline]
     pub fn with_ref<R>(&self, f: impl FnOnce(&[u8]) -> R) -> R {
         match self {
-            Self::String(s) => f(s.as_str().as_bytes()),
+            Self::String(s) => f(s.as_bytes()),
             Self::Buffer(buffer) => buffer.with_ref(f),
         }
     }

vm/src/function/fspath.rs

@@ -5,7 +5,7 @@ use crate::{
     function::PyStr,
     protocol::PyBuffer,
 };
-use std::{ffi::OsStr, path::PathBuf};
+use std::{borrow::Cow, ffi::OsStr, path::PathBuf};
 
 #[derive(Clone)]
 pub enum FsPath {
@@ -26,7 +26,7 @@ impl FsPath {
         let match1 = |obj: PyObjectRef| {
             let pathlike = match_class!(match obj {
                 s @ PyStr => {
-                    check_nul(s.as_str().as_bytes())?;
+                    check_nul(s.as_bytes())?;
                     FsPath::Str(s)
                 }
                 b @ PyBytes => {
@@ -58,26 +58,26 @@ impl FsPath {
         })
     }
 
-    pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<&OsStr> {
+    pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<Cow<'_, OsStr>> {
         // TODO: FS encodings
         match self {
-            FsPath::Str(s) => Ok(s.as_str().as_ref()),
-            FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm),
+            FsPath::Str(s) => vm.fsencode(s),
+            FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm).map(Cow::Borrowed),
         }
     }
 
     pub fn as_bytes(&self) -> &[u8] {
         // TODO: FS encodings
         match self {
-            FsPath::Str(s) => s.as_str().as_bytes(),
+            FsPath::Str(s) => s.as_bytes(),
             FsPath::Bytes(b) => b.as_bytes(),
         }
     }
 
-    pub fn as_str(&self) -> &str {
+    pub fn to_string_lossy(&self) -> Cow<'_, str> {
         match self {
-            FsPath::Bytes(b) => std::str::from_utf8(b).unwrap(),
-            FsPath::Str(s) => s.as_str(),
+            FsPath::Str(s) => s.to_string_lossy(),
+            FsPath::Bytes(s) => String::from_utf8_lossy(s),
         }
     }
 

vm/src/import.rs

@@ -204,15 +204,15 @@ fn remove_importlib_frames_inner(
 
 // TODO: This function should do nothing on verbose mode.
 // TODO: Fix this function after making PyTraceback.next mutable
-pub fn remove_importlib_frames(
-    vm: &VirtualMachine,
-    exc: &PyBaseExceptionRef,
-) -> PyBaseExceptionRef {
+pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyBaseExceptionRef) {
+    if vm.state.settings.verbose != 0 {
+        return;
+    }
+
     let always_trim = exc.fast_isinstance(vm.ctx.exceptions.import_error);
 
     if let Some(tb) = exc.traceback() {
         let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0;
         exc.set_traceback(trimmed_tb);
     }
-    exc.clone()
 }

vm/src/ospath.rs

@@ -21,28 +21,14 @@ pub(super) enum OutputMode {
 }
 
 impl OutputMode {
-    pub(super) fn process_path(self, path: impl Into<PathBuf>, vm: &VirtualMachine) -> PyResult {
-        fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyResult {
-            let path_as_string = |p: PathBuf| {
-                p.into_os_string().into_string().map_err(|_| {
-                    vm.new_unicode_decode_error(
-                        "Can't convert OS path to valid UTF-8 string".into(),
-                    )
-                })
-            };
+    pub(super) fn process_path(self, path: impl Into<PathBuf>, vm: &VirtualMachine) -> PyObjectRef {
+        fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyObjectRef {
             match mode {
-                OutputMode::String => path_as_string(path).map(|s| vm.ctx.new_str(s).into()),
-                OutputMode::Bytes => {
-                    #[cfg(any(unix, target_os = "wasi"))]
-                    {
-                        use rustpython_common::os::ffi::OsStringExt;
-                        Ok(vm.ctx.new_bytes(path.into_os_string().into_vec()).into())
-                    }
-                    #[cfg(windows)]
-                    {
-                        path_as_string(path).map(|s| vm.ctx.new_bytes(s.into_bytes()).into())
-                    }
-                }
+                OutputMode::String => vm.fsdecode(path).into(),
+                OutputMode::Bytes => vm
+                    .ctx
+                    .new_bytes(path.into_os_string().into_encoded_bytes())
+                    .into(),
             }
         }
         inner(self, path.into(), vm)
@@ -59,7 +45,7 @@ impl OsPath {
     }
 
     pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult<OsPath> {
-        let path = fspath.as_os_str(vm)?.to_owned();
+        let path = fspath.as_os_str(vm)?.into_owned();
         let mode = match fspath {
             FsPath::Str(_) => OutputMode::String,
             FsPath::Bytes(_) => OutputMode::Bytes,
@@ -88,7 +74,7 @@ impl OsPath {
         widestring::WideCString::from_os_str(&self.path).map_err(|err| err.to_pyexception(vm))
     }
 
-    pub fn filename(&self, vm: &VirtualMachine) -> PyResult {
+    pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef {
         self.mode.process_path(self.path.clone(), vm)
     }
 }
@@ -133,7 +119,7 @@ impl From<OsPath> for OsPathOrFd {
 impl OsPathOrFd {
     pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef {
         match self {
-            OsPathOrFd::Path(path) => path.filename(vm).unwrap_or_else(|_| vm.ctx.none()),
+            OsPathOrFd::Path(path) => path.filename(vm),
             OsPathOrFd::Fd(fd) => vm.ctx.new_int(*fd).into(),
         }
     }

vm/src/stdlib/builtins.rs

@@ -31,7 +31,9 @@ mod builtins {
         stdlib::sys,
         types::PyComparisonOp,
     };
+    use itertools::Itertools;
     use num_traits::{Signed, ToPrimitive};
+    use rustpython_common::wtf8::CodePoint;
 
     #[cfg(not(feature = "rustpython-compiler"))]
     const CODEGEN_NOT_SUPPORTED: &str =
@@ -85,13 +87,13 @@ mod builtins {
     }
 
     #[pyfunction]
-    fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<String> {
+    fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<CodePoint> {
         let value = i
             .try_to_primitive::<isize>(vm)?
             .to_u32()
-            .and_then(char::from_u32)
+            .and_then(CodePoint::from_u32)
             .ok_or_else(|| vm.new_value_error("chr() arg not in range(0x110000)".to_owned()))?;
-        Ok(value.to_string())
+        Ok(value)
     }
 
     #[derive(FromArgs)]
@@ -153,7 +155,7 @@ mod builtins {
                     return ast::compile(
                         vm,
                         args.source,
-                        args.filename.as_str(),
+                        &args.filename.to_string_lossy(),
                         mode,
                         Some(optimize),
                     );
@@ -202,7 +204,7 @@ mod builtins {
                             .compile_with_opts(
                                 source,
                                 mode,
-                                args.filename.as_str().to_owned(),
+                                args.filename.to_string_lossy().into_owned(),
                                 opts,
                             )
                             .map_err(|err| (err, Some(source)).to_pyexception(vm))?;
@@ -618,21 +620,15 @@ mod builtins {
                 }
                 Ok(u32::from(bytes[0]))
             }),
-            Either::B(string) => {
-                let string = string.as_str();
-                let string_len = string.chars().count();
-                if string_len != 1 {
-                    return Err(vm.new_type_error(format!(
+            Either::B(string) => match string.as_wtf8().code_points().exactly_one() {
+                Ok(character) => Ok(character.to_u32()),
+                Err(_) => {
+                    let string_len = string.char_len();
+                    Err(vm.new_type_error(format!(
                         "ord() expected a character, but string of length {string_len} found"
-                    )));
+                    )))
                 }
-                match string.chars().next() {
-                    Some(character) => Ok(character as u32),
-                    None => Err(vm.new_type_error(
-                        "ord() could not guess the integer representing this character".to_owned(),
-                    )),
-                }
-            }
+            },
         }
     }
 

vm/src/stdlib/codecs.rs

@@ -3,6 +3,8 @@ pub(crate) use _codecs::make_module;
 #[pymodule]
 mod _codecs {
     use crate::common::encodings;
+    use crate::common::str::StrKind;
+    use crate::common::wtf8::{Wtf8, Wtf8Buf};
     use crate::{
         AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine,
         builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple},
@@ -103,8 +105,8 @@ mod _codecs {
         }
     }
     impl encodings::StrBuffer for PyStrRef {
-        fn is_ascii(&self) -> bool {
-            PyStr::is_ascii(self)
+        fn is_compatible_with(&self, kind: StrKind) -> bool {
+            self.kind() <= kind
         }
     }
     impl encodings::ErrorHandler for ErrorsHandler<'_> {
@@ -114,7 +116,7 @@ mod _codecs {
 
         fn handle_encode_error(
             &self,
-            data: &str,
+            data: &Wtf8,
             char_range: Range<usize>,
             reason: &str,
         ) -> PyResult<(encodings::EncodeReplace<PyStrRef, PyBytesRef>, usize)> {
@@ -217,7 +219,7 @@ mod _codecs {
 
         fn error_encoding(
             &self,
-            data: &str,
+            data: &Wtf8,
             char_range: Range<usize>,
             reason: &str,
         ) -> Self::Error {
@@ -249,15 +251,15 @@ mod _codecs {
         #[inline]
         fn encode<'a, F>(self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult
         where
-            F: FnOnce(&str, &ErrorsHandler<'a>) -> PyResult<Vec<u8>>,
+            F: FnOnce(&Wtf8, &ErrorsHandler<'a>) -> PyResult<Vec<u8>>,
         {
             let errors = ErrorsHandler::new(name, self.errors, vm);
-            let encoded = encode(self.s.as_str(), &errors)?;
+            let encoded = encode(self.s.as_wtf8(), &errors)?;
             Ok((encoded, self.s.char_len()))
         }
     }
 
-    type DecodeResult = PyResult<(String, usize)>;
+    type DecodeResult = PyResult<(Wtf8Buf, usize)>;
 
     #[derive(FromArgs)]
     struct DecodeArgs {
@@ -310,6 +312,14 @@ mod _codecs {
 
     #[pyfunction]
     fn utf_8_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult {
+        if args.s.is_utf8()
+            || args
+                .errors
+                .as_ref()
+                .is_some_and(|s| s.is(identifier!(vm, surrogatepass)))
+        {
+            return Ok((args.s.as_bytes().to_vec(), args.s.byte_len()));
+        }
         do_codec!(utf8::encode, args, vm)
     }
 
@@ -321,7 +331,7 @@ mod _codecs {
     #[pyfunction]
     fn latin_1_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult {
         if args.s.is_ascii() {
-            return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len()));
+            return Ok((args.s.as_bytes().to_vec(), args.s.byte_len()));
         }
         do_codec!(latin_1::encode, args, vm)
     }
@@ -334,7 +344,7 @@ mod _codecs {
     #[pyfunction]
     fn ascii_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult {
         if args.s.is_ascii() {
-            return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len()));
+            return Ok((args.s.as_bytes().to_vec(), args.s.byte_len()));
         }
         do_codec!(ascii::encode, args, vm)
     }

vm/src/stdlib/io.rs

@@ -129,6 +129,7 @@ mod _io {
             PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard,
             PyThreadMutex, PyThreadMutexGuard,
         },
+        common::wtf8::{Wtf8, Wtf8Buf},
         convert::ToPyObject,
         function::{
             ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FuncArgs, IntoFuncArgs,
@@ -1909,10 +1910,12 @@ mod _io {
     impl Newlines {
         /// returns position where the new line starts if found, otherwise position at which to
         /// continue the search after more is read into the buffer
-        fn find_newline(&self, s: &str) -> Result<usize, usize> {
+        fn find_newline(&self, s: &Wtf8) -> Result<usize, usize> {
             let len = s.len();
             match self {
-                Newlines::Universal | Newlines::Lf => s.find('\n').map(|p| p + 1).ok_or(len),
+                Newlines::Universal | Newlines::Lf => {
+                    s.find("\n".as_ref()).map(|p| p + 1).ok_or(len)
+                }
                 Newlines::Passthrough => {
                     let bytes = s.as_bytes();
                     memchr::memchr2(b'\n', b'\r', bytes)
@@ -1927,7 +1930,7 @@ mod _io {
                         })
                         .ok_or(len)
                 }
-                Newlines::Cr => s.find('\n').map(|p| p + 1).ok_or(len),
+                Newlines::Cr => s.find("\n".as_ref()).map(|p| p + 1).ok_or(len),
                 Newlines::Crlf => {
                     // s[searched..] == remaining
                     let mut searched = 0;
@@ -1992,10 +1995,10 @@ mod _io {
             }
         }
 
-        fn len_str(s: &str) -> Self {
+        fn len_str(s: &Wtf8) -> Self {
             Utf8size {
                 bytes: s.len(),
-                chars: s.chars().count(),
+                chars: s.code_points().count(),
             }
         }
     }
@@ -2082,7 +2085,7 @@ mod _io {
     impl PendingWrite {
         fn as_bytes(&self) -> &[u8] {
             match self {
-                Self::Utf8(s) => s.as_str().as_bytes(),
+                Self::Utf8(s) => s.as_bytes(),
                 Self::Bytes(b) => b.as_bytes(),
             }
         }
@@ -2222,8 +2225,8 @@ mod _io {
             *data = None;
 
             let encoding = match args.encoding {
-                None if vm.state.settings.utf8_mode > 0 => PyStr::from("utf-8").into_ref(&vm.ctx),
-                Some(enc) if enc.as_str() != "locale" => enc,
+                None if vm.state.settings.utf8_mode > 0 => identifier!(vm, utf_8).to_owned(),
+                Some(enc) if enc.as_wtf8() != "locale" => enc,
                 _ => {
                     // None without utf8_mode or "locale" encoding
                     vm.import("locale", 0)?
@@ -2235,7 +2238,7 @@ mod _io {
 
             let errors = args
                 .errors
-                .unwrap_or_else(|| PyStr::from("strict").into_ref(&vm.ctx));
+                .unwrap_or_else(|| identifier!(vm, strict).to_owned());
 
             let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some();
             let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?;
@@ -2533,9 +2536,10 @@ mod _io {
                 *snapshot = Some((cookie.dec_flags, input_chunk.clone()));
                 let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?;
                 let decoded = check_decoded(decoded, vm)?;
-                let pos_is_valid = decoded
-                    .as_str()
-                    .is_char_boundary(cookie.bytes_to_skip as usize);
+                let pos_is_valid = crate::common::wtf8::is_code_point_boundary(
+                    decoded.as_wtf8(),
+                    cookie.bytes_to_skip as usize,
+                );
                 textio.set_decoded_chars(Some(decoded));
                 if !pos_is_valid {
                     return Err(vm.new_os_error("can't restore logical file position".to_owned()));
@@ -2714,9 +2718,9 @@ mod _io {
                 } else if chunks.len() == 1 {
                     chunks.pop().unwrap()
                 } else {
-                    let mut ret = String::with_capacity(chunks_bytes);
+                    let mut ret = Wtf8Buf::with_capacity(chunks_bytes);
                     for chunk in chunks {
-                        ret.push_str(chunk.as_str())
+                        ret.push_wtf8(chunk.as_wtf8())
                     }
                     PyStr::from(ret).into_ref(&vm.ctx)
                 }
@@ -2743,7 +2747,7 @@ mod _io {
 
             let char_len = obj.char_len();
 
-            let data = obj.as_str();
+            let data = obj.as_wtf8();
 
             let replace_nl = match textio.newline {
                 Newlines::Lf => Some("\n"),
@@ -2752,11 +2756,12 @@ mod _io {
                 Newlines::Universal if cfg!(windows) => Some("\r\n"),
                 _ => None,
             };
-            let has_lf = (replace_nl.is_some() || textio.line_buffering) && data.contains('\n');
-            let flush = textio.line_buffering && (has_lf || data.contains('\r'));
+            let has_lf = (replace_nl.is_some() || textio.line_buffering)
+                && data.contains_code_point('\n'.into());
+            let flush = textio.line_buffering && (has_lf || data.contains_code_point('\r'.into()));
             let chunk = if let Some(replace_nl) = replace_nl {
                 if has_lf {
-                    PyStr::from(data.replace('\n', replace_nl)).into_ref(&vm.ctx)
+                    PyStr::from(data.replace("\n".as_ref(), replace_nl.as_ref())).into_ref(&vm.ctx)
                 } else {
                     obj
                 }
@@ -2833,7 +2838,7 @@ mod _io {
                     if self.is_full_slice() {
                         self.0.char_len()
                     } else {
-                        self.slice().chars().count()
+                        self.slice().code_points().count()
                     }
                 }
                 #[inline]
@@ -2841,8 +2846,8 @@ mod _io {
                     self.1.len() >= self.0.byte_len()
                 }
                 #[inline]
-                fn slice(&self) -> &str {
-                    &self.0.as_str()[self.1.clone()]
+                fn slice(&self) -> &Wtf8 {
+                    &self.0.as_wtf8()[self.1.clone()]
                 }
                 #[inline]
                 fn slice_pystr(self, vm: &VirtualMachine) -> PyStrRef {
@@ -2893,7 +2898,7 @@ mod _io {
                     Some(remaining) => {
                         assert_eq!(textio.decoded_chars_used.bytes, 0);
                         offset_to_buffer = remaining.utf8_len();
-                        let decoded_chars = decoded_chars.as_str();
+                        let decoded_chars = decoded_chars.as_wtf8();
                         let line = if remaining.is_full_slice() {
                             let mut line = remaining.0;
                             line.concat_in_place(decoded_chars, vm);
@@ -2901,16 +2906,16 @@ mod _io {
                         } else {
                             let remaining = remaining.slice();
                             let mut s =
-                                String::with_capacity(remaining.len() + decoded_chars.len());
-                            s.push_str(remaining);
-                            s.push_str(decoded_chars);
+                                Wtf8Buf::with_capacity(remaining.len() + decoded_chars.len());
+                            s.push_wtf8(remaining);
+                            s.push_wtf8(decoded_chars);
                             PyStr::from(s).into_ref(&vm.ctx)
                         };
                         start = Utf8size::default();
                         line
                     }
                 };
-                let line_from_start = &line.as_str()[start.bytes..];
+                let line_from_start = &line.as_wtf8()[start.bytes..];
                 let nl_res = textio.newline.find_newline(line_from_start);
                 match nl_res {
                     Ok(p) | Err(p) => {
@@ -2921,7 +2926,7 @@ mod _io {
                                 endpos = start
                                     + Utf8size {
                                         chars: limit - chunked.chars,
-                                        bytes: crate::common::str::char_range_end(
+                                        bytes: crate::common::str::codepoint_range_end(
                                             line_from_start,
                                             limit - chunked.chars,
                                         )
@@ -2962,9 +2967,9 @@ mod _io {
                     chunked += cur_line.byte_len();
                     chunks.push(cur_line);
                 }
-                let mut s = String::with_capacity(chunked);
+                let mut s = Wtf8Buf::with_capacity(chunked);
                 for chunk in chunks {
-                    s.push_str(chunk.slice())
+                    s.push_wtf8(chunk.slice())
                 }
                 PyStr::from(s).into_ref(&vm.ctx)
             } else if let Some(cur_line) = cur_line {
@@ -3099,7 +3104,7 @@ mod _io {
                 return None;
             }
             let decoded_chars = self.decoded_chars.as_ref()?;
-            let avail = &decoded_chars.as_str()[self.decoded_chars_used.bytes..];
+            let avail = &decoded_chars.as_wtf8()[self.decoded_chars_used.bytes..];
             if avail.is_empty() {
                 return None;
             }
@@ -3111,7 +3116,7 @@ mod _io {
                     (PyStr::from(avail).into_ref(&vm.ctx), avail_chars)
                 }
             } else {
-                let s = crate::common::str::get_chars(avail, 0..n);
+                let s = crate::common::str::get_codepoints(avail, 0..n);
                 (PyStr::from(s).into_ref(&vm.ctx), n)
             };
             self.decoded_chars_used += Utf8size {
@@ -3141,11 +3146,11 @@ mod _io {
                 return decoded_chars;
             }
             // TODO: in-place editing of `str` when refcount == 1
-            let decoded_chars_unused = &decoded_chars.as_str()[chars_pos..];
-            let mut s = String::with_capacity(decoded_chars_unused.len() + append_len);
-            s.push_str(decoded_chars_unused);
+            let decoded_chars_unused = &decoded_chars.as_wtf8()[chars_pos..];
+            let mut s = Wtf8Buf::with_capacity(decoded_chars_unused.len() + append_len);
+            s.push_wtf8(decoded_chars_unused);
             if let Some(append) = append {
-                s.push_str(append.as_str())
+                s.push_wtf8(append.as_wtf8())
             }
             PyStr::from(s).into_ref(&vm.ctx)
         }
@@ -3305,14 +3310,14 @@ mod _io {
             };
             let orig_output: PyStrRef = output.try_into_value(vm)?;
             // this being Cow::Owned means we need to allocate a new string
-            let mut output = Cow::Borrowed(orig_output.as_str());
+            let mut output = Cow::Borrowed(orig_output.as_wtf8());
             if self.pendingcr && (final_ || !output.is_empty()) {
-                output = ["\r", &*output].concat().into();
+                output.to_mut().insert(0, '\r'.into());
                 self.pendingcr = false;
             }
             if !final_ {
-                if let Some(s) = output.strip_suffix('\r') {
-                    output = s.to_owned().into();
+                if let Some(s) = output.strip_suffix("\r".as_ref()) {
+                    output = Cow::Owned(s.to_owned());
                     self.pendingcr = true;
                 }
             }
@@ -3321,19 +3326,21 @@ mod _io {
                 return Ok(vm.ctx.empty_str.to_owned());
             }
 
-            if (self.seennl == SeenNewline::LF || self.seennl.is_empty()) && !output.contains('\r')
+            if (self.seennl == SeenNewline::LF || self.seennl.is_empty())
+                && !output.contains_code_point('\r'.into())
             {
-                if self.seennl.is_empty() && output.contains('\n') {
+                if self.seennl.is_empty() && output.contains_code_point('\n'.into()) {
                     self.seennl.insert(SeenNewline::LF);
                 }
             } else if !self.translate {
-                let mut matches = output.match_indices(['\r', '\n']);
+                let output = output.as_bytes();
+                let mut matches = memchr::memchr2_iter(b'\r', b'\n', output);
                 while !self.seennl.is_all() {
-                    let Some((i, c)) = matches.next() else { break };
-                    match c {
-                        "\n" => self.seennl.insert(SeenNewline::LF),
+                    let Some(i) = matches.next() else { break };
+                    match output[i] {
+                        b'\n' => self.seennl.insert(SeenNewline::LF),
                         // if c isn't \n, it can only be \r
-                        _ if output[i + 1..].starts_with('\n') => {
+                        _ if output.get(i + 1) == Some(&b'\n') => {
                             matches.next();
                             self.seennl.insert(SeenNewline::CRLF);
                         }
@@ -3341,30 +3348,31 @@ mod _io {
                     }
                 }
             } else {
-                let mut chunks = output.match_indices(['\r', '\n']);
-                let mut new_string = String::with_capacity(output.len());
+                let bytes = output.as_bytes();
+                let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes);
+                let mut new_string = Wtf8Buf::with_capacity(output.len());
                 let mut last_modification_index = 0;
-                while let Some((cr_index, chunk)) = chunks.next() {
-                    if chunk == "\r" {
+                while let Some(cr_index) = matches.next() {
+                    if bytes[cr_index] == b'\r' {
                         // skip copying the CR
                         let mut next_chunk_index = cr_index + 1;
-                        if output[cr_index + 1..].starts_with('\n') {
-                            chunks.next();
+                        if bytes.get(cr_index + 1) == Some(&b'\n') {
+                            matches.next();
                             self.seennl.insert(SeenNewline::CRLF);
                             // skip the LF too
                             next_chunk_index += 1;
                         } else {
                             self.seennl.insert(SeenNewline::CR);
                         }
-                        new_string.push_str(&output[last_modification_index..cr_index]);
-                        new_string.push('\n');
+                        new_string.push_wtf8(&output[last_modification_index..cr_index]);
+                        new_string.push_char('\n');
                         last_modification_index = next_chunk_index;
                     } else {
                         self.seennl.insert(SeenNewline::LF);
                     }
                 }
-                new_string.push_str(&output[last_modification_index..]);
-                output = new_string.into();
+                new_string.push_wtf8(&output[last_modification_index..]);
+                output = Cow::Owned(new_string);
             }
 
             Ok(match output {
@@ -3404,7 +3412,7 @@ mod _io {
         ) -> PyResult {
             let raw_bytes = object
                 .flatten()
-                .map_or_else(Vec::new, |v| v.as_str().as_bytes().to_vec());
+                .map_or_else(Vec::new, |v| v.as_bytes().to_vec());
 
             StringIO {
                 buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))),
@@ -3453,7 +3461,7 @@ mod _io {
         // write string to underlying vector
         #[pymethod]
         fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult<u64> {
-            let bytes = data.as_str().as_bytes();
+            let bytes = data.as_bytes();
             self.buffer(vm)?
                 .write(bytes)
                 .ok_or_else(|| vm.new_type_error("Error Writing String".to_owned()))

vm/src/stdlib/nt.rs

@@ -249,7 +249,7 @@ pub(crate) mod module {
             .as_ref()
             .canonicalize()
             .map_err(|e| e.to_pyexception(vm))?;
-        path.mode.process_path(real, vm)
+        Ok(path.mode.process_path(real, vm))
     }
 
     #[pyfunction]
@@ -282,7 +282,7 @@ pub(crate) mod module {
             }
         }
         let buffer = widestring::WideCString::from_vec_truncate(buffer);
-        path.mode.process_path(buffer.to_os_string(), vm)
+        Ok(path.mode.process_path(buffer.to_os_string(), vm))
     }
 
     #[pyfunction]
@@ -297,7 +297,7 @@ pub(crate) mod module {
             return Err(errno_err(vm));
         }
         let buffer = widestring::WideCString::from_vec_truncate(buffer);
-        path.mode.process_path(buffer.to_os_string(), vm)
+        Ok(path.mode.process_path(buffer.to_os_string(), vm))
     }
 
     #[pyfunction]

vm/src/stdlib/operator.rs

@@ -328,7 +328,7 @@ mod _operator {
                         "comparing strings with non-ASCII characters is not supported".to_owned(),
                     ));
                 }
-                cmp::timing_safe_cmp(a.as_str().as_bytes(), b.as_str().as_bytes())
+                cmp::timing_safe_cmp(a.as_bytes(), b.as_bytes())
             }
             (Either::B(a), Either::B(b)) => {
                 a.with_ref(|a| b.with_ref(|b| cmp::timing_safe_cmp(a, b)))

vm/src/stdlib/os.rs

@@ -332,7 +332,7 @@ pub(super) mod _os {
                 };
                 dir_iter
                     .map(|entry| match entry {
-                        Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm),
+                        Ok(entry_path) => Ok(path.mode.process_path(entry_path.file_name(), vm)),
                         Err(err) => Err(IOErrorBuilder::with_filename(&err, path.clone(), vm)),
                     })
                     .collect::<PyResult<_>>()?
@@ -352,22 +352,18 @@ pub(super) mod _os {
                     let mut dir =
                         nix::dir::Dir::from_fd(new_fd).map_err(|e| e.into_pyexception(vm))?;
                     dir.iter()
-                        .filter_map(|entry| {
-                            entry
-                                .map_err(|e| e.into_pyexception(vm))
-                                .and_then(|entry| {
-                                    let fname = entry.file_name().to_bytes();
-                                    Ok(match fname {
-                                        b"." | b".." => None,
-                                        _ => Some(
-                                            OutputMode::String
-                                                .process_path(ffi::OsStr::from_bytes(fname), vm)?,
-                                        ),
-                                    })
-                                })
-                                .transpose()
+                        .filter_map_ok(|entry| {
+                            let fname = entry.file_name().to_bytes();
+                            match fname {
+                                b"." | b".." => None,
+                                _ => Some(
+                                    OutputMode::String
+                                        .process_path(ffi::OsStr::from_bytes(fname), vm),
+                                ),
+                            }
                         })
-                        .collect::<PyResult<_>>()?
+                        .collect::<Result<_, _>>()
+                        .map_err(|e| e.into_pyexception(vm))?
                 }
             }
         };
@@ -376,7 +372,7 @@ pub(super) mod _os {
 
     fn env_bytes_as_bytes(obj: &Either<PyStrRef, PyBytesRef>) -> &[u8] {
         match obj {
-            Either::A(s) => s.as_str().as_bytes(),
+            Either::A(s) => s.as_bytes(),
             Either::B(b) => b.as_bytes(),
         }
     }
@@ -429,7 +425,7 @@ pub(super) mod _os {
         let [] = dir_fd.0;
         let path =
             fs::read_link(&path).map_err(|err| IOErrorBuilder::with_filename(&err, path, vm))?;
-        mode.process_path(path, vm)
+        Ok(mode.process_path(path, vm))
     }
 
     #[pyattr]
@@ -452,12 +448,12 @@ pub(super) mod _os {
     impl DirEntry {
         #[pygetset]
         fn name(&self, vm: &VirtualMachine) -> PyResult {
-            self.mode.process_path(&self.file_name, vm)
+            Ok(self.mode.process_path(&self.file_name, vm))
         }
 
         #[pygetset]
         fn path(&self, vm: &VirtualMachine) -> PyResult {
-            self.mode.process_path(&self.pathval, vm)
+            Ok(self.mode.process_path(&self.pathval, vm))
         }
 
         fn perform_on_metadata(
@@ -908,12 +904,12 @@ pub(super) mod _os {
 
     #[pyfunction]
     fn getcwd(vm: &VirtualMachine) -> PyResult {
-        OutputMode::String.process_path(curdir_inner(vm)?, vm)
+        Ok(OutputMode::String.process_path(curdir_inner(vm)?, vm))
     }
 
     #[pyfunction]
     fn getcwdb(vm: &VirtualMachine) -> PyResult {
-        OutputMode::Bytes.process_path(curdir_inner(vm)?, vm)
+        Ok(OutputMode::Bytes.process_path(curdir_inner(vm)?, vm))
     }
 
     #[pyfunction]

vm/src/stdlib/sre.rs

@@ -9,6 +9,7 @@ mod _sre {
             PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr,
             PyStrRef, PyTuple, PyTupleRef, PyTypeRef,
         },
+        common::wtf8::{Wtf8, Wtf8Buf},
         common::{ascii, hash::PyHash},
         convert::ToPyObject,
         function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue},
@@ -66,10 +67,15 @@ mod _sre {
         }
     }
 
-    impl SreStr for &str {
+    impl SreStr for &Wtf8 {
         fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef {
             vm.ctx
-                .new_str(self.chars().take(end).skip(start).collect::<String>())
+                .new_str(
+                    self.code_points()
+                        .take(end)
+                        .skip(start)
+                        .collect::<Wtf8Buf>(),
+                )
                 .into()
         }
     }
@@ -206,12 +212,12 @@ mod _sre {
     impl Pattern {
         fn with_str<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R>
         where
-            F: FnOnce(&str) -> PyResult<R>,
+            F: FnOnce(&Wtf8) -> PyResult<R>,
         {
             let string = string.payload::<PyStr>().ok_or_else(|| {
                 vm.new_type_error(format!("expected string got '{}'", string.class()))
             })?;
-            f(string.as_str())
+            f(string.as_wtf8())
         }
 
         fn with_bytes<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R>
@@ -425,7 +431,7 @@ mod _sre {
                 let is_template = if zelf.isbytes {
                     Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))?
                 } else {
-                    Self::with_str(&repl, vm, |x| Ok(x.contains('\\')))?
+                    Self::with_str(&repl, vm, |x| Ok(x.contains("\\".as_ref())))?
                 };
 
                 if is_template {

vm/src/stdlib/string.rs

@@ -9,6 +9,7 @@ mod _string {
     use crate::common::format::{
         FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
     };
+    use crate::common::wtf8::{CodePoint, Wtf8Buf};
     use crate::{
         PyObjectRef, PyResult, VirtualMachine,
         builtins::{PyList, PyStrRef},
@@ -18,10 +19,10 @@ mod _string {
     use std::mem;
 
     fn create_format_part(
-        literal: String,
-        field_name: Option<String>,
-        format_spec: Option<String>,
-        conversion_spec: Option<char>,
+        literal: Wtf8Buf,
+        field_name: Option<Wtf8Buf>,
+        format_spec: Option<Wtf8Buf>,
+        conversion_spec: Option<CodePoint>,
         vm: &VirtualMachine,
     ) -> PyObjectRef {
         let tuple = (
@@ -36,10 +37,10 @@ mod _string {
     #[pyfunction]
     fn formatter_parser(text: PyStrRef, vm: &VirtualMachine) -> PyResult<PyList> {
         let format_string =
-            FormatString::from_str(text.as_str()).map_err(|e| e.to_pyexception(vm))?;
+            FormatString::from_str(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?;
 
-        let mut result = Vec::new();
-        let mut literal = String::new();
+        let mut result: Vec<PyObjectRef> = Vec::new();
+        let mut literal = Wtf8Buf::new();
         for part in format_string.format_parts {
             match part {
                 FormatPart::Field {
@@ -55,7 +56,7 @@ mod _string {
                         vm,
                     ));
                 }
-                FormatPart::Literal(text) => literal.push_str(&text),
+                FormatPart::Literal(text) => literal.push_wtf8(&text),
             }
         }
         if !literal.is_empty() {
@@ -75,7 +76,7 @@ mod _string {
         text: PyStrRef,
         vm: &VirtualMachine,
     ) -> PyResult<(PyObjectRef, PyList)> {
-        let field_name = FieldName::parse(text.as_str()).map_err(|e| e.to_pyexception(vm))?;
+        let field_name = FieldName::parse(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?;
 
         let first = match field_name.field_type {
             FieldType::Auto => vm.ctx.new_str(ascii!("")).into(),

vm/src/stdlib/sys.rs

@@ -458,21 +458,13 @@ mod sys {
     }
 
     #[pyfunction]
-    fn getfilesystemencoding(_vm: &VirtualMachine) -> String {
-        // TODO: implement non-utf-8 mode.
-        "utf-8".to_owned()
+    fn getfilesystemencoding(vm: &VirtualMachine) -> PyStrRef {
+        vm.fs_encoding().to_owned()
     }
 
-    #[cfg(not(windows))]
     #[pyfunction]
-    fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String {
-        "surrogateescape".to_owned()
-    }
-
-    #[cfg(windows)]
-    #[pyfunction]
-    fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String {
-        "surrogatepass".to_owned()
+    fn getfilesystemencodeerrors(vm: &VirtualMachine) -> PyStrRef {
+        vm.fs_encode_errors().to_owned()
     }
 
     #[pyfunction]

vm/src/vm/context.rs

@@ -51,7 +51,7 @@ pub struct Context {
 }
 
 macro_rules! declare_const_name {
-    ($($name:ident,)*) => {
+    ($($name:ident$(: $s:literal)?,)*) => {
         #[derive(Debug, Clone, Copy)]
         #[allow(non_snake_case)]
         pub struct ConstName {
@@ -61,11 +61,13 @@ macro_rules! declare_const_name {
         impl ConstName {
             unsafe fn new(pool: &StringPool, typ: &PyTypeRef) -> Self {
                 Self {
-                    $($name: unsafe { pool.intern(stringify!($name), typ.clone()) },)*
+                    $($name: unsafe { pool.intern(declare_const_name!(@string $name $($s)?), typ.clone()) },)*
                 }
             }
         }
-    }
+    };
+    (@string $name:ident) => { stringify!($name) };
+    (@string $name:ident $string:literal) => { $string };
 }
 
 declare_const_name! {
@@ -236,6 +238,15 @@ declare_const_name! {
     flush,
     close,
     WarningMessage,
+    strict,
+    ignore,
+    replace,
+    xmlcharrefreplace,
+    backslashreplace,
+    namereplace,
+    surrogatepass,
+    surrogateescape,
+    utf_8: "utf-8",
 }
 
 // Basic objects:

vm/src/vm/interpreter.rs

@@ -163,7 +163,7 @@ mod tests {
             let b = vm.new_pyobj(4_i32);
             let res = vm._mul(&a, &b).unwrap();
             let value = res.payload::<PyStr>().unwrap();
-            assert_eq!(value.as_ref(), "Hello Hello Hello Hello ")
+            assert_eq!(value.as_str(), "Hello Hello Hello Hello ")
         })
     }
 }

vm/src/vm/mod.rs

@@ -41,11 +41,12 @@ use nix::{
     sys::signal::{SaFlags, SigAction, SigSet, Signal::SIGINT, kill, sigaction},
     unistd::getpid,
 };
-use std::sync::atomic::AtomicBool;
 use std::{
     borrow::Cow,
     cell::{Cell, Ref, RefCell},
     collections::{HashMap, HashSet},
+    ffi::{OsStr, OsString},
+    sync::atomic::AtomicBool,
 };
 
 pub use context::Context;
@@ -610,7 +611,7 @@ impl VirtualMachine {
                 let from_list = from_list.to_pyobject(self);
                 import_func
                     .call((module.to_owned(), globals, locals, from_list, level), self)
-                    .map_err(|exc| import::remove_importlib_frames(self, &exc))
+                    .inspect_err(|exc| import::remove_importlib_frames(self, exc))
             }
         }
     }
@@ -901,6 +902,54 @@ impl VirtualMachine {
         run_module_as_main.call((module,), self)?;
         Ok(())
     }
+
+    pub fn fs_encoding(&self) -> &'static PyStrInterned {
+        identifier!(self, utf_8)
+    }
+
+    pub fn fs_encode_errors(&self) -> &'static PyStrInterned {
+        if cfg!(windows) {
+            identifier!(self, surrogatepass)
+        } else {
+            identifier!(self, surrogateescape)
+        }
+    }
+
+    pub fn fsdecode(&self, s: impl Into<OsString>) -> PyStrRef {
+        match s.into().into_string() {
+            Ok(s) => self.ctx.new_str(s),
+            Err(s) => {
+                let bytes = self.ctx.new_bytes(s.into_encoded_bytes());
+                let errors = self.fs_encode_errors().to_owned();
+                let res = self.state.codec_registry.decode_text(
+                    bytes.into(),
+                    "utf-8",
+                    Some(errors),
+                    self,
+                );
+                self.expect_pyresult(res, "fsdecode should be lossless and never fail")
+            }
+        }
+    }
+
+    pub fn fsencode<'a>(&self, s: &'a Py<PyStr>) -> PyResult<Cow<'a, OsStr>> {
+        if cfg!(windows) || s.is_utf8() {
+            // XXX: this is sketchy on windows; it's not guaranteed that the
+            //      OsStr encoding will always be compatible with WTF-8.
+            let s = unsafe { OsStr::from_encoded_bytes_unchecked(s.as_bytes()) };
+            return Ok(Cow::Borrowed(s));
+        }
+        let errors = self.fs_encode_errors().to_owned();
+        let bytes = self
+            .state
+            .codec_registry
+            .encode_text(s.to_owned(), "utf-8", Some(errors), self)?
+            .to_vec();
+        // XXX: this is sketchy on windows; it's not guaranteed that the
+        //      OsStr encoding will always be compatible with WTF-8.
+        let s = unsafe { OsString::from_encoded_bytes_unchecked(bytes) };
+        Ok(Cow::Owned(s))
+    }
 }
 
 impl AsRef<Context> for VirtualMachine {

vm/sre_engine/Cargo.toml

@@ -15,6 +15,7 @@ name = "benches"
 harness = false
 
 [dependencies]
+rustpython-common = { workspace = true }
 num_enum = { workspace = true }
 bitflags = { workspace = true }
 optional = "0.5"

vm/sre_engine/src/string.rs

@@ -1,3 +1,5 @@
+use rustpython_common::wtf8::Wtf8;
+
 #[derive(Debug, Clone, Copy)]
 pub struct StringCursor {
     pub(crate) ptr: *const u8,
@@ -148,6 +150,72 @@ impl StrDrive for &str {
     }
 }
 
+impl StrDrive for &Wtf8 {
+    #[inline]
+    fn count(&self) -> usize {
+        self.code_points().count()
+    }
+
+    #[inline]
+    fn create_cursor(&self, n: usize) -> StringCursor {
+        let mut cursor = StringCursor {
+            ptr: self.as_bytes().as_ptr(),
+            position: 0,
+        };
+        Self::skip(&mut cursor, n);
+        cursor
+    }
+
+    #[inline]
+    fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) {
+        if cursor.ptr.is_null() || cursor.position > n {
+            *cursor = Self::create_cursor(self, n);
+        } else if cursor.position < n {
+            Self::skip(cursor, n - cursor.position);
+        }
+    }
+
+    #[inline]
+    fn advance(cursor: &mut StringCursor) -> u32 {
+        cursor.position += 1;
+        unsafe { next_code_point(&mut cursor.ptr) }
+    }
+
+    #[inline]
+    fn peek(cursor: &StringCursor) -> u32 {
+        let mut ptr = cursor.ptr;
+        unsafe { next_code_point(&mut ptr) }
+    }
+
+    #[inline]
+    fn skip(cursor: &mut StringCursor, n: usize) {
+        cursor.position += n;
+        for _ in 0..n {
+            unsafe { next_code_point(&mut cursor.ptr) };
+        }
+    }
+
+    #[inline]
+    fn back_advance(cursor: &mut StringCursor) -> u32 {
+        cursor.position -= 1;
+        unsafe { next_code_point_reverse(&mut cursor.ptr) }
+    }
+
+    #[inline]
+    fn back_peek(cursor: &StringCursor) -> u32 {
+        let mut ptr = cursor.ptr;
+        unsafe { next_code_point_reverse(&mut ptr) }
+    }
+
+    #[inline]
+    fn back_skip(cursor: &mut StringCursor, n: usize) {
+        cursor.position -= n;
+        for _ in 0..n {
+            unsafe { next_code_point_reverse(&mut cursor.ptr) };
+        }
+    }
+}
+
 /// Reads the next code point out of a byte iterator (assuming a
 /// UTF-8-like encoding).
 ///