mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Luan Taraschi
79cf1d9dd0
unicodeescape_string is a port of CPython 2.7's
PyUnicode_EncodeUnicodeEscape, where the branch mapping a character at or
above 0x10000 to \U00xxxxxx sits under #ifdef Py_UNICODE_WIDE and the
branch folding a surrogate pair sits under the #else. The port kept the
"# ifdef Py_UNICODE_WIDE" and "# endif" comments around the first one but
dropped the #else, so both branches run.
The pairing branch reads the next character without checking that there
is one, so a high surrogate at the end of the string walks off it:
>>> "\ud800".encode("unicode_escape")
IndexError: index out of range # CPython: b'\ud800'
And when there is a next character it folds the two, which no wide build
does:
>>> "𐀀".encode("unicode_escape")
b'\U00010000' # CPython: b'\ud800\udc00'
With the branch gone a high surrogate falls through to the 16-bit case
and comes out as \udXXX, which is where a low surrogate already went.
test_incremental_surrogatepass in UnicodeEscapeTest carried an
expectedFailure for the IndexError. It passes now, so the override goes
with it.
Assisted-by: Claude Code:claude-opus-5
44 lines
1.4 KiB
Python
44 lines
1.4 KiB
Python
from testutils import assert_raises
|
||
|
||
try:
|
||
b" \xff".decode("ascii")
|
||
except UnicodeDecodeError as e:
|
||
assert e.start == 3
|
||
assert e.end == 4
|
||
else:
|
||
assert False, "should have thrown UnicodeDecodeError"
|
||
|
||
assert_raises(UnicodeEncodeError, "¿como estás?".encode, "ascii")
|
||
|
||
|
||
def round_trip(s, encoding="utf-8"):
|
||
encoded = s.encode(encoding)
|
||
decoded = encoded.decode(encoding)
|
||
assert s == decoded
|
||
|
||
|
||
round_trip("👺♦ 𝐚Şđƒ ☆☝")
|
||
round_trip("☢🐣 ᖇ𝓤𝕊тⓟ𝕐𝕥卄σ𝔫 ♬👣")
|
||
round_trip("💀👌 ק𝔂tℍⓞ𝓷 ３ 🔥👤")
|
||
|
||
# Bytes should not assume an encoding for isupper/islower
|
||
assert "Æ".isupper()
|
||
assert not "Æ".encode().isupper()
|
||
assert "æ".islower()
|
||
assert not "æ".encode().islower()
|
||
|
||
# Invalid Unicode
|
||
assert not b"\x80\x80".islower()
|
||
assert not b"\x80\x80".isupper()
|
||
assert b"\x80cat\x80".islower()
|
||
assert b"\x80CAT\x80".isupper()
|
||
|
||
# A lone surrogate gets an escape of its own wherever it sits, and a high one
|
||
# followed by a low one stays two escapes rather than being folded into one.
|
||
assert "\ud800".encode("unicode_escape") == b"\\ud800"
|
||
assert "a\ud800".encode("unicode_escape") == b"a\\ud800"
|
||
assert "\ud800b".encode("unicode_escape") == b"\\ud800b"
|
||
assert "\ud800\ud800".encode("unicode_escape") == b"\\ud800\\ud800"
|
||
assert "\ud800\udc00".encode("unicode_escape") == b"\\ud800\\udc00"
|
||
assert "\U00010000".encode("unicode_escape") == b"\\U00010000"
|