From 2fbd01f23e10a4916a0aeeee9ede26d5c67bd586 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 25 Jul 2022 00:06:03 +0900 Subject: [PATCH] Fix surrogateescape_errors --- Lib/test/test_cmd_line_script.py | 4 ---- Lib/test/test_difflib.py | 2 -- Lib/test/test_httpservers.py | 2 -- Lib/test/test_tarfile.py | 8 -------- Lib/test/test_utf8_mode.py | 6 ++++-- vm/src/codecs.rs | 32 ++++++++++++++++---------------- vm/src/stdlib/codecs.rs | 3 ++- 7 files changed, 22 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index c12ef182f..88845d26b 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -602,10 +602,6 @@ class CmdLineTest(unittest.TestCase): 'stdout=%r stderr=%r' % (stdout, stderr)) self.assertEqual(0, rc) - # TODO: RUSTPYTHON - if sys.platform == "linux": - test_non_ascii = unittest.expectedFailure(test_non_ascii) - def test_issue20500_exit_with_exception_value(self): script = textwrap.dedent("""\ import sys diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 50cfdedae..68da83dda 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -350,8 +350,6 @@ class TestBytes(unittest.TestCase): line, bytes, "all lines of diff should be bytes, but got: %r" % line) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_content(self): # if we receive byte strings, we return byte strings a = [b'hello', b'andr\xe9'] # iso-8859-1 bytes diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 28a1ec372..28392ccec 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -387,8 +387,6 @@ class SimpleHTTPServerTestCase(BaseTestCase): reader.close() return body - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == 'darwin', 'undecodable name cannot always be decoded on macOS') @unittest.skipIf(sys.platform == 'win32', diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index ad52049af..10b4fe9f8 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -488,8 +488,6 @@ class MiscReadTestBase(CommonReadTest): with self.assertRaisesRegex(ValueError, 'mode must be '): tar = self.taropen(tmpname, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fileobj_with_offset(self): # Skip the first member and store values from the second member # of the testtar. @@ -676,8 +674,6 @@ class MiscReadTestBase(CommonReadTest): finally: os_helper.unlink(empty) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parallel_iteration(self): # Issue #16601: Restarting iteration over tarfile continued # from where it left off. @@ -2339,16 +2335,12 @@ class CommandLineTest(unittest.TestCase): self.assertEqual(out, b'') self.assertNotEqual(err.strip(), b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_test_command(self): for tar_name in testtarnames: for opt in '-t', '--test': out = self.tarfilecmd(opt, tar_name) self.assertEqual(out, b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_test_command_verbose(self): for tar_name in testtarnames: for opt in '-v', '--verbose': diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index e5e956870..429206dd5 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -222,8 +222,6 @@ class UTF8ModeTests(unittest.TestCase): out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc) self.assertEqual(out, 'UTF-8 UTF-8') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(MS_WINDOWS, 'test specific to Unix') def test_cmd_line(self): arg = 'h\xe9\u20ac'.encode('utf-8') @@ -251,6 +249,10 @@ class UTF8ModeTests(unittest.TestCase): with self.subTest(LC_ALL=loc): check('utf8=0', [c_arg], LC_ALL=loc) + # TODO: RUSTPYTHON + if sys.platform == "linux": + test_cmd_line = unittest.expectedFailure(test_cmd_line) + def test_optim_level(self): # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag # twice when -X utf8 requires to parse the configuration twice (when diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index bda17e334..23c5c8920 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -661,34 +661,34 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Stri } } -fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { +fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; + let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; let s_after_start = - crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); - let num_chars = range.len(); - let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars).map(|x| x as u32) { - if !(0xd800..=0xdfff).contains(&c) { + crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or(""); + let mut out: Vec = Vec::with_capacity(range.len()); + for ch in s_after_start.chars().take(range.len()) { + let ch = ch as u32; + if !(0xdc80..=0xdcff).contains(&ch) { // Not a UTF-8b surrogate, fail with original exception return Err(err.downcast().unwrap()); } - write!(out, "#{}", c - 0xdc00).unwrap(); + out.push((ch - 0xdc00) as u8); } - Ok((out, range.end)) + let out = vm.ctx.new_bytes(out); + Ok((out.into(), range.end)) } else if is_decode_err(&err, vm) { let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..) - .unwrap_or("") - .as_bytes(); + let object = err.get_attr("object", vm)?; + let object = PyBytesRef::try_from_object(vm, object)?; + let p = &object.as_bytes()[range.clone()]; let mut consumed = 0; let mut replace = String::with_capacity(4 * range.len()); while consumed < 4 && consumed < range.len() { - let c = s_after_start[consumed] as u32; + let c = p[consumed] as u32; + // Refuse to escape ASCII bytes if c < 128 { - // Refuse to escape ASCII bytes break; } write!(replace, "#{}", 0xdc00 + c).unwrap(); @@ -697,7 +697,7 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(St if consumed == 0 { return Err(err.downcast().unwrap()); } - Ok((replace, range.start + consumed)) + Ok((vm.new_pyobj(replace), range.start + consumed)) } else { Err(bad_err_type(err, vm)) } diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 9de8fd1a4..478b0a752 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -173,7 +173,8 @@ mod _codecs { vm.ctx.new_str(reason).into(), ], ); - let res = vm.invoke(self.handler_func()?, (decode_exc.clone(),))?; + let handler = self.handler_func()?; + let res = vm.invoke(handler, (decode_exc.clone(),))?; let new_data = decode_exc .get_arg(1) .ok_or_else(|| vm.new_type_error("object attribute not set".to_owned()))?;