Fix surrogateescape_errors

Lib/test/test_cmd_line_script.py

@@ -602,10 +602,6 @@ class CmdLineTest(unittest.TestCase):
             'stdout=%r stderr=%r' % (stdout, stderr))
         self.assertEqual(0, rc)
 
-    # TODO: RUSTPYTHON
-    if sys.platform == "linux":
-        test_non_ascii = unittest.expectedFailure(test_non_ascii)
-
     def test_issue20500_exit_with_exception_value(self):
         script = textwrap.dedent("""\
             import sys

Lib/test/test_difflib.py

@@ -350,8 +350,6 @@ class TestBytes(unittest.TestCase):
                 line, bytes,
                 "all lines of diff should be bytes, but got: %r" % line)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_byte_content(self):
         # if we receive byte strings, we return byte strings
         a = [b'hello', b'andr\xe9']     # iso-8859-1 bytes

Lib/test/test_httpservers.py

@@ -387,8 +387,6 @@ class SimpleHTTPServerTestCase(BaseTestCase):
         reader.close()
         return body
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @unittest.skipIf(sys.platform == 'darwin',
                      'undecodable name cannot always be decoded on macOS')
     @unittest.skipIf(sys.platform == 'win32',

Lib/test/test_tarfile.py

@@ -488,8 +488,6 @@ class MiscReadTestBase(CommonReadTest):
         with self.assertRaisesRegex(ValueError, 'mode must be '):
             tar = self.taropen(tmpname, '')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_fileobj_with_offset(self):
         # Skip the first member and store values from the second member
         # of the testtar.
@@ -676,8 +674,6 @@ class MiscReadTestBase(CommonReadTest):
         finally:
             os_helper.unlink(empty)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_parallel_iteration(self):
         # Issue #16601: Restarting iteration over tarfile continued
         # from where it left off.
@@ -2339,16 +2335,12 @@ class CommandLineTest(unittest.TestCase):
         self.assertEqual(out, b'')
         self.assertNotEqual(err.strip(), b'')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_test_command(self):
         for tar_name in testtarnames:
             for opt in '-t', '--test':
                 out = self.tarfilecmd(opt, tar_name)
                 self.assertEqual(out, b'')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_test_command_verbose(self):
         for tar_name in testtarnames:
             for opt in '-v', '--verbose':

Lib/test/test_utf8_mode.py

@@ -222,8 +222,6 @@ class UTF8ModeTests(unittest.TestCase):
                 out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc)
                 self.assertEqual(out, 'UTF-8 UTF-8')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @unittest.skipIf(MS_WINDOWS, 'test specific to Unix')
     def test_cmd_line(self):
         arg = 'h\xe9\u20ac'.encode('utf-8')
@@ -251,6 +249,10 @@ class UTF8ModeTests(unittest.TestCase):
             with self.subTest(LC_ALL=loc):
                 check('utf8=0', [c_arg], LC_ALL=loc)
 
+    # TODO: RUSTPYTHON
+    if sys.platform == "linux":
+        test_cmd_line = unittest.expectedFailure(test_cmd_line)
+
     def test_optim_level(self):
         # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag
         # twice when -X utf8 requires to parse the configuration twice (when

vm/src/codecs.rs

@@ -661,34 +661,34 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Stri
     }
 }
 
-fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> {
+fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> {
     if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) {
         let range = extract_unicode_error_range(&err, vm)?;
-        let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
+        let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
         let s_after_start =
-            crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
-        let num_chars = range.len();
-        let mut out = String::with_capacity(num_chars * 4);
-        for c in s_after_start.chars().take(num_chars).map(|x| x as u32) {
-            if !(0xd800..=0xdfff).contains(&c) {
+            crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or("");
+        let mut out: Vec<u8> = Vec::with_capacity(range.len());
+        for ch in s_after_start.chars().take(range.len()) {
+            let ch = ch as u32;
+            if !(0xdc80..=0xdcff).contains(&ch) {
                 // Not a UTF-8b surrogate, fail with original exception
                 return Err(err.downcast().unwrap());
             }
-            write!(out, "#{}", c - 0xdc00).unwrap();
+            out.push((ch - 0xdc00) as u8);
         }
-        Ok((out, range.end))
+        let out = vm.ctx.new_bytes(out);
+        Ok((out.into(), range.end))
     } else if is_decode_err(&err, vm) {
         let range = extract_unicode_error_range(&err, vm)?;
-        let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
-        let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..)
-            .unwrap_or("")
-            .as_bytes();
+        let object = err.get_attr("object", vm)?;
+        let object = PyBytesRef::try_from_object(vm, object)?;
+        let p = &object.as_bytes()[range.clone()];
         let mut consumed = 0;
         let mut replace = String::with_capacity(4 * range.len());
         while consumed < 4 && consumed < range.len() {
-            let c = s_after_start[consumed] as u32;
+            let c = p[consumed] as u32;
+            // Refuse to escape ASCII bytes
             if c < 128 {
-                // Refuse to escape ASCII bytes
                 break;
             }
             write!(replace, "#{}", 0xdc00 + c).unwrap();
@@ -697,7 +697,7 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(St
         if consumed == 0 {
             return Err(err.downcast().unwrap());
         }
-        Ok((replace, range.start + consumed))
+        Ok((vm.new_pyobj(replace), range.start + consumed))
     } else {
         Err(bad_err_type(err, vm))
     }

vm/src/stdlib/codecs.rs

@@ -173,7 +173,8 @@ mod _codecs {
                     vm.ctx.new_str(reason).into(),
                 ],
             );
-            let res = vm.invoke(self.handler_func()?, (decode_exc.clone(),))?;
+            let handler = self.handler_func()?;
+            let res = vm.invoke(handler, (decode_exc.clone(),))?;
             let new_data = decode_exc
                 .get_arg(1)
                 .ok_or_else(|| vm.new_type_error("object attribute not set".to_owned()))?;