Drop the narrow-build surrogate pairing from unicode_escape (#8569)

unicodeescape_string is a port of CPython 2.7's
PyUnicode_EncodeUnicodeEscape, where the branch mapping a character at or
above 0x10000 to \U00xxxxxx sits under #ifdef Py_UNICODE_WIDE and the
branch folding a surrogate pair sits under the #else. The port kept the
"# ifdef Py_UNICODE_WIDE" and "# endif" comments around the first one but
dropped the #else, so both branches run.

The pairing branch reads the next character without checking that there
is one, so a high surrogate at the end of the string walks off it:

    >>> "\ud800".encode("unicode_escape")
    IndexError: index out of range      # CPython: b'\ud800'

And when there is a next character it folds the two, which no wide build
does:

    >>> "𐀀".encode("unicode_escape")
    b'\U00010000'                      # CPython: b'\ud800\udc00'

With the branch gone a high surrogate falls through to the 16-bit case
and comes out as \udXXX, which is where a low surrogate already went.

test_incremental_surrogatepass in UnicodeEscapeTest carried an
expectedFailure for the IndexError. It passes now, so the override goes
with it.

Assisted-by: Claude Code:claude-opus-5

Lib/_pycodecs.py

@@ -1102,20 +1102,6 @@ def unicodeescape_string(s, size, quotes):
             pos += 1
             continue
         # endif
-        # /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */
-        elif ord(ch) >= 0xD800 and ord(ch) < 0xDC00:
-            pos += 1
-            ch2 = s[pos]
-
-            if ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF:
-                ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000
-                p.append(b"\\U%08x" % ucs)
-                pos += 1
-                continue
-
-            # /* Fall through: isolated surrogates are copied as-is */
-            pos -= 1
-
         # /* Map 16-bit characters to '\uxxxx' */
         if ord(ch) >= 256:
             p.append(b"\\u%04x" % ord(ch))

Lib/test/test_codecs.py

@@ -2776,9 +2776,6 @@ class UnicodeEscapeTest(ReadTest, unittest.TestCase):
             ]
         )
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; IndexError: index out of range
-    def test_incremental_surrogatepass(self):
-        return super().test_incremental_surrogatepass()
 
 class RawUnicodeEscapeTest(ReadTest, unittest.TestCase):
     encoding = "raw-unicode-escape"

extra_tests/snippets/builtin_str_encode.py

@@ -32,3 +32,12 @@ assert not b"\x80\x80".islower()
 assert not b"\x80\x80".isupper()
 assert b"\x80cat\x80".islower()
 assert b"\x80CAT\x80".isupper()
+
+# A lone surrogate gets an escape of its own wherever it sits, and a high one
+# followed by a low one stays two escapes rather than being folded into one.
+assert "\ud800".encode("unicode_escape") == b"\\ud800"
+assert "a\ud800".encode("unicode_escape") == b"a\\ud800"
+assert "\ud800b".encode("unicode_escape") == b"\\ud800b"
+assert "\ud800\ud800".encode("unicode_escape") == b"\\ud800\\ud800"
+assert "\ud800\udc00".encode("unicode_escape") == b"\\ud800\\udc00"
+assert "\U00010000".encode("unicode_escape") == b"\\U00010000"