mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-29 03:04:38 +08:00
youknowdot
64330b80b9
* Implement native Unicode 17 grapheme iteration Add the CPython extended grapheme state machine, full-codepoint properties, and traced Segment/iterator types with positional index handling. Refresh canonical unicodedata tests through update_lib while retaining pending markers. Include a losslessly range-compressed exact-final property oracle and differential test. Source-reviewed implementation; candidate runtime validation remains pending. Reference property generation completed against the pinned CPython Unicode extension. Assisted-by: OpenAI Codex:model version unavailable * Bind grapheme Segment methods to the Python object Use the required Py<Segment> receiver for native descriptors; preserve payload ownership and representation behavior. Assisted-by: OpenAI Codex:model version unavailable * Use explicit object receivers for Segment descriptors Keep the inherent impl on the local payload and receive &Py<Self>, as required for stdlib classes outside the VM crate. Assisted-by: OpenAI Codex:model version unavailable * Mark grapheme property lookups as must-use Satisfy strict Clippy for the three pure property functions without changing their behavior. Assisted-by: OpenAI Codex:model version unavailable * Enable validated Unicode grapheme canonical tests Remove seven stale expected-failure markers through update_lib metadata after native property parity, canonical methods, full Unicode 17 grapheme corpus, and reference-cycle GC validation. Assisted-by: OpenAI Codex:model version unavailable * Accept platform line endings in grapheme oracle header Compare the complete provenance line without its line ending so the same exact Unicode 17 reference is accepted after Windows checkout conversion. Keep the complete code-point property sweep unchanged. Assisted-by: OpenAI Codex:model version unavailable * Keep grapheme property regression in test module Move the existing complete Unicode 17 property regression into the existing cfg(test) module to satisfy strict Clippy. Preserve the fixture, assertions, code-point coverage, and platform-independent provenance line check unchanged. Assisted-by: OpenAI Codex:model version unavailable * Qualify grapheme iterator return type for generated bindings Assisted-by: OpenAI Codex:model version unavailable * Fix deprecated PyO3 code input import in benchmarks Use the canonical code module path introduced in PyO3 0.30 without suppressing deprecation warnings. Assisted-by: OpenAI Codex:model version unavailable
1445 lines
64 KiB
Python
Vendored
1445 lines
64 KiB
Python
Vendored
""" Tests for the unicodedata module.
|
||
|
||
Written by Marc-Andre Lemburg (mal@lemburg.com).
|
||
|
||
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
|
||
|
||
"""
|
||
|
||
from functools import partial
|
||
import hashlib
|
||
from http.client import HTTPException
|
||
import sys
|
||
import unicodedata
|
||
import unittest
|
||
import weakref
|
||
from test.support import (
|
||
gc_collect,
|
||
open_urlresource,
|
||
requires_resource,
|
||
script_helper,
|
||
cpython_only,
|
||
check_disallow_instantiation,
|
||
force_not_colorized,
|
||
is_resource_enabled,
|
||
findfile,
|
||
)
|
||
|
||
|
||
quicktest = not is_resource_enabled('cpu')
|
||
|
||
def iterallchars():
|
||
maxunicode = 0xffff if quicktest else sys.maxunicode
|
||
return map(chr, range(maxunicode + 1))
|
||
|
||
|
||
def check_version(testfile):
|
||
hdr = testfile.readline()
|
||
return unicodedata.unidata_version in hdr
|
||
|
||
|
||
def download_test_data_file(filename):
|
||
TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{filename}"
|
||
|
||
try:
|
||
return open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version)
|
||
except PermissionError:
|
||
raise unittest.SkipTest(
|
||
f"Permission error when downloading {TESTDATAURL} "
|
||
f"into the test data directory"
|
||
)
|
||
except (OSError, HTTPException) as exc:
|
||
raise unittest.SkipTest(f"Failed to download {TESTDATAURL}: {exc}")
|
||
|
||
|
||
class UnicodeMethodsTest(unittest.TestCase):
|
||
|
||
# update this, if the database changes
|
||
expectedchecksum = ('47a99fa654ef1f50e89d2e9697b7b041fccb5a05'
|
||
if quicktest else
|
||
'8b2615a9fc627676cbc0b6fac0191177df97ef5f')
|
||
|
||
def test_method_checksum(self):
|
||
h = hashlib.sha1()
|
||
for char in iterallchars():
|
||
s1 = char + 'abc'
|
||
s2 = char + 'ABC'
|
||
s3 = char + '123'
|
||
data = (
|
||
# Predicates (single char)
|
||
"01"[char.isalnum()],
|
||
"01"[char.isalpha()],
|
||
"01"[char.isdecimal()],
|
||
"01"[char.isdigit()],
|
||
"01"[char.islower()],
|
||
"01"[char.isnumeric()],
|
||
"01"[char.isspace()],
|
||
"01"[char.istitle()],
|
||
"01"[char.isupper()],
|
||
|
||
# Predicates (multiple chars)
|
||
"01"[s1.isalnum()],
|
||
"01"[s1.isalpha()],
|
||
"01"[s3.isdecimal()],
|
||
"01"[s3.isdigit()],
|
||
"01"[s1.islower()],
|
||
"01"[s3.isnumeric()],
|
||
"01"[(char + ' \t').isspace()],
|
||
"01"[s1.istitle()],
|
||
"01"[s2.isupper()],
|
||
|
||
# Mappings (single char)
|
||
char.lower(),
|
||
char.upper(),
|
||
char.title(),
|
||
|
||
# Mappings (multiple chars)
|
||
s1.lower(),
|
||
s2.upper(),
|
||
s1.title(),
|
||
s2.title(),
|
||
|
||
)
|
||
h.update(''.join(data).encode('utf-8', 'surrogatepass'))
|
||
result = h.hexdigest()
|
||
self.assertEqual(result, self.expectedchecksum)
|
||
|
||
|
||
class BaseUnicodeFunctionsTest:
|
||
|
||
@unittest.skip("TODO: RUSTPYTHON; AssertionError: '7bda75e48a961a01ab328358980cefc5c0a1666d' != '68cd01e2c680b851c1fcab012efb5635'")
|
||
def test_function_checksum(self):
|
||
db = self.db
|
||
data = []
|
||
h = hashlib.sha1()
|
||
|
||
for char in iterallchars():
|
||
data = "%.12g%.12g%.12g%s%s%s%s%s%s%s" % (
|
||
# Properties
|
||
db.digit(char, -1),
|
||
db.numeric(char, -1),
|
||
db.decimal(char, -1),
|
||
db.category(char),
|
||
db.bidirectional(char),
|
||
db.decomposition(char),
|
||
db.mirrored(char),
|
||
db.combining(char),
|
||
db.east_asian_width(char),
|
||
db.name(char, ""),
|
||
)
|
||
h.update(data.encode("ascii"))
|
||
result = h.hexdigest()
|
||
self.assertEqual(result, self.expectedchecksum)
|
||
|
||
def test_name(self):
|
||
name = self.db.name
|
||
self.assertRaises(ValueError, name, '\0')
|
||
self.assertRaises(ValueError, name, '\n')
|
||
self.assertRaises(ValueError, name, '\x1F')
|
||
self.assertRaises(ValueError, name, '\x7F')
|
||
self.assertRaises(ValueError, name, '\x9F')
|
||
self.assertRaises(ValueError, name, '\uFFFE')
|
||
self.assertRaises(ValueError, name, '\uFFFF')
|
||
self.assertRaises(ValueError, name, '\U0010FFFF')
|
||
self.assertEqual(name('\U0010FFFF', 42), 42)
|
||
|
||
self.assertEqual(name(' '), 'SPACE')
|
||
self.assertEqual(name('1'), 'DIGIT ONE')
|
||
self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A')
|
||
self.assertEqual(name('\xA0'), 'NO-BREAK SPACE')
|
||
self.assertEqual(name('\u0221', None), None if self.old else
|
||
'LATIN SMALL LETTER D WITH CURL')
|
||
self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400')
|
||
self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5')
|
||
self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA')
|
||
self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH')
|
||
self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900')
|
||
self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A')
|
||
self.assertEqual(name('\uFBF9'),
|
||
'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA '
|
||
'ABOVE WITH ALEF MAKSURA ISOLATED FORM')
|
||
self.assertEqual(name('\U00013460', None), None if self.old else
|
||
'EGYPTIAN HIEROGLYPH-13460')
|
||
self.assertEqual(name('\U000143FA', None), None if self.old else
|
||
'EGYPTIAN HIEROGLYPH-143FA')
|
||
self.assertEqual(name('\U00017000', None), None if self.old else
|
||
'TANGUT IDEOGRAPH-17000')
|
||
self.assertEqual(name('\U00018B00', None), None if self.old else
|
||
'KHITAN SMALL SCRIPT CHARACTER-18B00')
|
||
self.assertEqual(name('\U00018CD5', None), None if self.old else
|
||
'KHITAN SMALL SCRIPT CHARACTER-18CD5')
|
||
self.assertEqual(name('\U00018CFF', None), None if self.old else
|
||
'KHITAN SMALL SCRIPT CHARACTER-18CFF')
|
||
self.assertEqual(name('\U00018D1E', None), None if self.old else
|
||
'TANGUT IDEOGRAPH-18D1E')
|
||
self.assertEqual(name('\U0001B170', None), None if self.old else
|
||
'NUSHU CHARACTER-1B170')
|
||
self.assertEqual(name('\U0001B2FB', None), None if self.old else
|
||
'NUSHU CHARACTER-1B2FB')
|
||
self.assertEqual(name('\U0001FBA8', None), None if self.old else
|
||
'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO '
|
||
'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE')
|
||
self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6')
|
||
self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D')
|
||
self.assertEqual(name('\U00033479', None), None if self.old else
|
||
'CJK UNIFIED IDEOGRAPH-33479')
|
||
|
||
@requires_resource('cpu')
|
||
def test_name_inverse_lookup(self):
|
||
for char in iterallchars():
|
||
looked_name = self.db.name(char, None)
|
||
if looked_name is not None:
|
||
self.assertEqual(self.db.lookup(looked_name), char)
|
||
|
||
def test_no_names_in_pua(self):
|
||
puas = [*range(0xe000, 0xf8ff),
|
||
*range(0xf0000, 0xfffff),
|
||
*range(0x100000, 0x10ffff)]
|
||
for i in puas:
|
||
char = chr(i)
|
||
self.assertRaises(ValueError, self.db.name, char)
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
|
||
def test_lookup_nonexistant(self):
|
||
# just make sure that lookup can fail
|
||
for nonexistent in [
|
||
"LATIN SMLL LETR A",
|
||
"OPEN HANDS SIGHS",
|
||
"DREGS",
|
||
"HANDBUG",
|
||
"MODIFIER LETTER CYRILLIC SMALL QUESTION MARK",
|
||
"???",
|
||
"CJK UNIFIED IDEOGRAPH-03400",
|
||
"CJK UNIFIED IDEOGRAPH-020000",
|
||
"CJK UNIFIED IDEOGRAPH-33FF",
|
||
"CJK UNIFIED IDEOGRAPH-F900",
|
||
"CJK UNIFIED IDEOGRAPH-13460",
|
||
"CJK UNIFIED IDEOGRAPH-17000",
|
||
"CJK UNIFIED IDEOGRAPH-18B00",
|
||
"CJK UNIFIED IDEOGRAPH-1B170",
|
||
"CJK COMPATIBILITY IDEOGRAPH-3400",
|
||
"TANGUT IDEOGRAPH-3400",
|
||
"HANGUL SYLLABLE AC00",
|
||
]:
|
||
self.assertRaises(KeyError, self.db.lookup, nonexistent)
|
||
|
||
def test_digit(self):
|
||
self.assertEqual(self.db.digit('A', None), None)
|
||
self.assertEqual(self.db.digit('9'), 9)
|
||
self.assertEqual(self.db.digit('\u215b', None), None)
|
||
self.assertEqual(self.db.digit('\u2468'), 9)
|
||
self.assertEqual(self.db.digit('\U00020000', None), None)
|
||
self.assertEqual(self.db.digit('\U0001D7FD'), 7)
|
||
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.digit('\U0001fbf9', None), 9)
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.digit('\U00016ac9', None), 9)
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.digit('\U0001e4f9', None), 9)
|
||
|
||
self.assertRaises(TypeError, self.db.digit)
|
||
self.assertRaises(TypeError, self.db.digit, 'xx')
|
||
self.assertRaises(ValueError, self.db.digit, 'x')
|
||
|
||
def test_numeric(self):
|
||
self.assertEqual(self.db.numeric('A',None), None)
|
||
self.assertEqual(self.db.numeric('9'), 9)
|
||
self.assertEqual(self.db.numeric('\u215b'), 0.125)
|
||
self.assertEqual(self.db.numeric('\u2468'), 9.0)
|
||
self.assertEqual(self.db.numeric('\U00020000', None), None)
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000)
|
||
# Changed in 4.1.0
|
||
self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None)
|
||
# New in 5.0.0
|
||
self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0)
|
||
# New in 5.1.0
|
||
self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0)
|
||
# Changed in 5.2.0
|
||
self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16)
|
||
# New in 6.0.0
|
||
self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25)
|
||
# New in 12.0.0
|
||
self.assertEqual(self.db.numeric('\U0001ed3c', None), None if self.old else 0.5)
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.numeric('\U0001fbf9', None), None if self.old else 9)
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.numeric('\U00016ac9', None), None if self.old else 9)
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.numeric('\U0001e4f9', None), None if self.old else 9)
|
||
|
||
self.assertRaises(TypeError, self.db.numeric)
|
||
self.assertRaises(TypeError, self.db.numeric, 'xx')
|
||
self.assertRaises(ValueError, self.db.numeric, 'x')
|
||
|
||
def test_decimal(self):
|
||
self.assertEqual(self.db.decimal('A',None), None)
|
||
self.assertEqual(self.db.decimal('9'), 9)
|
||
self.assertEqual(self.db.decimal('\u215b', None), None)
|
||
self.assertEqual(self.db.decimal('\u2468', None), None)
|
||
self.assertEqual(self.db.decimal('\U00020000', None), None)
|
||
self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.decimal('\xb2', None), 2 if self.old else None)
|
||
self.assertEqual(self.db.decimal('\u1369', None), 1 if self.old else None)
|
||
# New in 5.0.0
|
||
self.assertEqual(self.db.decimal('\u07c0', None), None if self.old else 0)
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.decimal('\U0001fbf9', None), None if self.old else 9)
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.decimal('\U00016ac9', None), None if self.old else 9)
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.decimal('\U0001e4f9', None), None if self.old else 9)
|
||
|
||
self.assertRaises(TypeError, self.db.decimal)
|
||
self.assertRaises(TypeError, self.db.decimal, 'xx')
|
||
self.assertRaises(ValueError, self.db.decimal, 'x')
|
||
|
||
def test_category(self):
|
||
self.assertEqual(self.db.category('\uFFFE'), 'Cn')
|
||
self.assertEqual(self.db.category('a'), 'Ll')
|
||
self.assertEqual(self.db.category('A'), 'Lu')
|
||
self.assertEqual(self.db.category('\U00020000'), 'Lo')
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.category('\U0001012A'), 'Cn' if self.old else 'No')
|
||
self.assertEqual(self.db.category('\U000e01ef'), 'Cn' if self.old else 'Mn')
|
||
# New in 5.1.0
|
||
self.assertEqual(self.db.category('\u0374'), 'Sk' if self.old else 'Lm')
|
||
# Changed in 13.0.0
|
||
self.assertEqual(self.db.category('\u0b55'), 'Cn' if self.old else 'Mn')
|
||
self.assertEqual(self.db.category('\U0003134a'), 'Cn' if self.old else 'Lo')
|
||
# Changed in 14.0.0
|
||
self.assertEqual(self.db.category('\u061d'), 'Cn' if self.old else 'Po')
|
||
self.assertEqual(self.db.category('\U0002b738'), 'Cn' if self.old else 'Lo')
|
||
# Changed in 15.0.0
|
||
self.assertEqual(self.db.category('\u0cf3'), 'Cn' if self.old else 'Mc')
|
||
self.assertEqual(self.db.category('\U000323af'), 'Cn' if self.old else 'Lo')
|
||
|
||
self.assertRaises(TypeError, self.db.category)
|
||
self.assertRaises(TypeError, self.db.category, 'xx')
|
||
|
||
def test_bidirectional(self):
|
||
self.assertEqual(self.db.bidirectional('\uFFFE'), '' if self.old else 'BN')
|
||
self.assertEqual(self.db.bidirectional(' '), 'WS')
|
||
self.assertEqual(self.db.bidirectional('A'), 'L')
|
||
self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.bidirectional('+'), 'ET' if self.old else 'ES')
|
||
self.assertEqual(self.db.bidirectional('\u0221'), '' if self.old else 'L')
|
||
self.assertEqual(self.db.bidirectional('\U000e01ef'), '' if self.old else 'NSM')
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.bidirectional('\u0b55'), '' if self.old else 'NSM')
|
||
self.assertEqual(self.db.bidirectional('\U0003134a'), '' if self.old else 'L')
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.bidirectional('\u061d'), '' if self.old else 'AL')
|
||
self.assertEqual(self.db.bidirectional('\U0002b738'), '' if self.old else 'L')
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.bidirectional('\u0cf3'), '' if self.old else 'L')
|
||
self.assertEqual(self.db.bidirectional('\U000323af'), '' if self.old else 'L')
|
||
# New in 16.0.0
|
||
self.assertEqual(self.db.bidirectional('\u0897'), '' if self.old else 'NSM')
|
||
self.assertEqual(self.db.bidirectional('\U0001fbef'), '' if self.old else 'ON')
|
||
# New in 17.0.0
|
||
self.assertEqual(self.db.bidirectional('\u088f'), '' if self.old else 'AL')
|
||
self.assertEqual(self.db.bidirectional('\U0001fbfa'), '' if self.old else 'ON')
|
||
|
||
self.assertRaises(TypeError, self.db.bidirectional)
|
||
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
|
||
|
||
def test_bidirectional_unassigned(self):
|
||
self.assertEqual(self.db.bidirectional('\u0378'), '' if self.old else 'L')
|
||
self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
|
||
self.assertEqual(self.db.bidirectional('\u20CF'), '' if self.old else 'ET')
|
||
self.assertEqual(self.db.bidirectional('\u0590'), '' if self.old else 'R')
|
||
self.assertEqual(self.db.bidirectional('\uFFFF'), '' if self.old else 'BN')
|
||
self.assertEqual(self.db.bidirectional('\U0001FFFE'), '' if self.old else 'BN')
|
||
self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '<compat> 03A3 != ''
|
||
def test_decomposition(self):
|
||
self.assertEqual(self.db.decomposition('\uFFFE'),'')
|
||
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.decomposition('\u03f9'), '' if self.old else '<compat> 03A3')
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.decomposition('\uab69'), '' if self.old else '<super> 028D')
|
||
self.assertEqual(self.db.decomposition('\U00011938'), '' if self.old else '11935 11930')
|
||
self.assertEqual(self.db.decomposition('\U0001fbf9'), '' if self.old else '<font> 0039')
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.decomposition('\ua7f2'), '' if self.old else '<super> 0043')
|
||
self.assertEqual(self.db.decomposition('\U000107ba'), '' if self.old else '<super> 1DF1E')
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else '<super> 04B1')
|
||
# New in 16.0.0
|
||
self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else '<font> 0041')
|
||
# New in 17.0.0
|
||
self.assertEqual(self.db.decomposition('\uA7F1'), '' if self.old else '<super> 0053')
|
||
|
||
# Hangul characters
|
||
self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161')
|
||
self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6')
|
||
self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161')
|
||
self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2')
|
||
|
||
self.assertRaises(TypeError, self.db.decomposition)
|
||
self.assertRaises(TypeError, self.db.decomposition, 'xx')
|
||
|
||
def test_mirrored(self):
|
||
self.assertEqual(self.db.mirrored('\uFFFE'), 0)
|
||
self.assertEqual(self.db.mirrored('a'), 0)
|
||
self.assertEqual(self.db.mirrored('\u2201'), 1)
|
||
self.assertEqual(self.db.mirrored('\U00020000'), 0)
|
||
|
||
# New in 5.0.0
|
||
self.assertEqual(self.db.mirrored('\u0f3a'), 0 if self.old else 1)
|
||
self.assertEqual(self.db.mirrored('\U0001d7c3'), 0 if self.old else 1)
|
||
# New in 11.0.0
|
||
self.assertEqual(self.db.mirrored('\u29a1'), 1 if self.old else 0)
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.mirrored('\u2e5c'), 0 if self.old else 1)
|
||
# New in 16.0.0
|
||
self.assertEqual(self.db.mirrored('\u226D'), 0 if self.old else 1)
|
||
|
||
self.assertRaises(TypeError, self.db.mirrored)
|
||
self.assertRaises(TypeError, self.db.mirrored, 'xx')
|
||
|
||
def test_combining(self):
|
||
self.assertEqual(self.db.combining('\uFFFE'), 0)
|
||
self.assertEqual(self.db.combining('a'), 0)
|
||
self.assertEqual(self.db.combining('\u20e1'), 230)
|
||
self.assertEqual(self.db.combining('\U00020000'), 0)
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.combining('\u0350'), 0 if self.old else 230)
|
||
# New in 9.0.0
|
||
self.assertEqual(self.db.combining('\U0001e94a'), 0 if self.old else 7)
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.combining('\u1abf'), 0 if self.old else 220)
|
||
self.assertEqual(self.db.combining('\U00016ff1'), 0 if self.old else 6)
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.combining('\u0c3c'), 0 if self.old else 7)
|
||
self.assertEqual(self.db.combining('\U0001e2ae'), 0 if self.old else 230)
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220)
|
||
# New in 16.0.0
|
||
self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230)
|
||
# New in 17.0.0
|
||
self.assertEqual(self.db.combining('\u1ACF'), 0 if self.old else 230)
|
||
|
||
self.assertRaises(TypeError, self.db.combining)
|
||
self.assertRaises(TypeError, self.db.combining, 'xx')
|
||
|
||
def test_normalization(self):
|
||
# Test normalize() and is_normalized()
|
||
def check(ch, expected):
|
||
if isinstance(expected, str):
|
||
expected = [expected]*4
|
||
forms = ('NFC', 'NFD', 'NFKC', 'NFKD')
|
||
result = [self.db.normalize(form, ch) for form in forms]
|
||
self.assertEqual(ascii(result), ascii(list(expected)))
|
||
self.assertEqual([self.db.is_normalized(form, ch) for form in forms],
|
||
[ch == y for x, y in zip(result, expected)])
|
||
|
||
check('', '')
|
||
check('A', 'A')
|
||
check(' ', ' ')
|
||
check('\U0010ffff', '\U0010ffff')
|
||
check('abc', 'abc')
|
||
# Broken in 4.0.0
|
||
check('\u0340', '\u0300')
|
||
check('\u0300', '\u0300')
|
||
check('\U0002fa1d', '\U0002a600')
|
||
check('\U0002a600', '\U0002a600')
|
||
check('\u0344', '\u0308\u0301')
|
||
check('\u0308\u0301', '\u0308\u0301')
|
||
# Broken in 4.0.0 and 4.0.1
|
||
check('\U0001d1bc', '\U0001d1ba\U0001d165')
|
||
check('\U0001d1ba\U0001d165', '\U0001d1ba\U0001d165')
|
||
check('\ufb2c', '\u05e9\u05bc\u05c1')
|
||
check('\u05e9\u05bc\u05c1', '\u05e9\u05bc\u05c1')
|
||
check('\U0001d1c0', '\U0001d1ba\U0001d165\U0001d16f')
|
||
check('\U0001d1ba\U0001d165\U0001d16f', '\U0001d1ba\U0001d165\U0001d16f')
|
||
|
||
# Broken in 4.0.0
|
||
check('\xa0', ['\xa0', '\xa0', ' ', ' '])
|
||
check('\u2003', ['\u2003', '\u2003', ' ', ' '])
|
||
check('\U0001d7ff', ['\U0001d7ff', '\U0001d7ff', '9', '9'])
|
||
|
||
check('\xa8', ['\xa8', '\xa8', ' \u0308', ' \u0308'])
|
||
check(' \u0308', ' \u0308')
|
||
|
||
check('\xc0', ['\xc0', 'A\u0300']*2)
|
||
check('A\u0300', ['\xc0', 'A\u0300']*2)
|
||
|
||
check('\ud7a3', ['\ud7a3', '\u1112\u1175\u11c2']*2)
|
||
check('\u1112\u1175\u11c2', ['\ud7a3', '\u1112\u1175\u11c2']*2)
|
||
|
||
check('\xb4', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
|
||
check('\u1ffd', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
|
||
check(' \u0301', ' \u0301')
|
||
|
||
check('\xc5', ['\xc5', 'A\u030a']*2)
|
||
check('\u212b', ['\xc5', 'A\u030a']*2)
|
||
check('A\u030a', ['\xc5', 'A\u030a']*2)
|
||
|
||
check('\u1f71', ['\u03ac', '\u03b1\u0301']*2)
|
||
check('\u03ac', ['\u03ac', '\u03b1\u0301']*2)
|
||
check('\u03b1\u0301', ['\u03ac', '\u03b1\u0301']*2)
|
||
|
||
check('\u01c4', ['\u01c4', '\u01c4', 'D\u017d', 'DZ\u030c'])
|
||
check('D\u017d', ['D\u017d', 'DZ\u030c']*2)
|
||
check('DZ\u030c', ['D\u017d', 'DZ\u030c']*2)
|
||
|
||
check('\u1fed', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
|
||
check('\xa8\u0300', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
|
||
check(' \u0308\u0300', ' \u0308\u0300')
|
||
|
||
check('\u326e', ['\u326e', '\u326e', '\uac00', '\u1100\u1161'])
|
||
check('\u320e', ['\u320e', '\u320e', '(\uac00)', '(\u1100\u1161)'])
|
||
check('(\uac00)', ['(\uac00)', '(\u1100\u1161)']*2)
|
||
check('(\u1100\u1161)', ['(\uac00)', '(\u1100\u1161)']*2)
|
||
|
||
check('\u0385', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
|
||
check('\u1fee', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
|
||
check('\xa8\u0301', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
|
||
check(' \u0308\u0301', ' \u0308\u0301')
|
||
|
||
check('\u1fdf', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
|
||
check('\u1ffe\u0342', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
|
||
check('\u1ffe', ['\u1ffe', '\u1ffe', ' \u0314', ' \u0314'])
|
||
check(' \u0314\u0342', ' \u0314\u0342')
|
||
|
||
check('\u03d3', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
|
||
check('\u03d2\u0301', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
|
||
check('\u038e', ['\u038e', '\u03a5\u0301']*2)
|
||
check('\u1feb', ['\u038e', '\u03a5\u0301']*2)
|
||
check('\u03a5\u0301', ['\u038e', '\u03a5\u0301']*2)
|
||
|
||
check('\u0626', ['\u0626', '\u064a\u0654']*2)
|
||
check('\u064a\u0654', ['\u0626', '\u064a\u0654']*2)
|
||
check('\ufe89', ['\ufe89', '\ufe89', '\u0626', '\u064a\u0654'])
|
||
check('\ufe8a', ['\ufe8a', '\ufe8a', '\u0626', '\u064a\u0654'])
|
||
check('\ufe8b', ['\ufe8b', '\ufe8b', '\u0626', '\u064a\u0654'])
|
||
check('\ufe8c', ['\ufe8c', '\ufe8c', '\u0626', '\u064a\u0654'])
|
||
|
||
check('\ufef9', ['\ufef9', '\ufef9', '\u0644\u0625', '\u0644\u0627\u0655'])
|
||
check('\ufefa', ['\ufefa', '\ufefa', '\u0644\u0625', '\u0644\u0627\u0655'])
|
||
check('\ufefb', ['\ufefb', '\ufefb', '\u0644\u0627', '\u0644\u0627'])
|
||
check('\ufefc', ['\ufefc', '\ufefc', '\u0644\u0627', '\u0644\u0627'])
|
||
check('\u0644\u0625', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
|
||
check('\u0644\u0627\u0655', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
|
||
check('\u0644\u0627', '\u0644\u0627')
|
||
|
||
# Broken in 4.0.0
|
||
check('\u327c', '\u327c' if self.old else
|
||
['\u327c', '\u327c', '\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169'])
|
||
check('\ucc38\uace0', ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
|
||
check('\ucc38', ['\ucc38', '\u110e\u1161\u11b7']*2)
|
||
check('\u110e\u1161\u11b7\u1100\u1169',
|
||
['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
|
||
check('\u110e\u1161\u11b7\u1100',
|
||
['\ucc38\u1100', '\u110e\u1161\u11b7\u1100']*2)
|
||
check('\u110e\u1161\u11b7',
|
||
['\ucc38', '\u110e\u1161\u11b7']*2)
|
||
check('\u110e\u1161',
|
||
['\ucc28', '\u110e\u1161']*2)
|
||
check('\u110e', '\u110e')
|
||
# Broken in 4.0.0-12.0.0
|
||
check('\U00011938', '\U00011938' if self.old else
|
||
['\U00011938', '\U00011935\U00011930']*2)
|
||
check('\U00011935\U00011930', ['\U00011938', '\U00011935\U00011930']*2)
|
||
# New in 4.0.1
|
||
check('\u321d', '\u321d' if self.old else
|
||
['\u321d', '\u321d', '(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)'])
|
||
check('(\uc624\uc804)',
|
||
['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
|
||
check('(\u110b\u1169\u110c\u1165\u11ab)',
|
||
['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
|
||
check('\u4d57', '\u4d57')
|
||
check('\u45d7', '\u45d7' if self.old else '\u45d7')
|
||
check('\U0002f9bf', '\u4d57' if self.old else '\u45d7')
|
||
# New in 4.1.0
|
||
check('\u03a3', '\u03a3')
|
||
check('\u03f9', '\u03f9' if self.old else
|
||
['\u03f9', '\u03f9', '\u03a3', '\u03a3'])
|
||
# New in 5.0.0
|
||
check('\u1b06', '\u1b06' if self.old else ['\u1b06', '\u1b05\u1b35']*2)
|
||
# New in 5.2.0
|
||
check('\U0001f213', '\U0001f213' if self.old else
|
||
['\U0001f213', '\U0001f213', '\u30c7', '\u30c6\u3099'])
|
||
# New in 6.1.0
|
||
check('\ufa2e', '\ufa2e' if self.old else '\u90de')
|
||
# New in 13.0.0
|
||
check('\U00011938', '\U00011938' if self.old else
|
||
['\U00011938', '\U00011935\U00011930', '\U00011938', '\U00011935\U00011930'])
|
||
check('\U0001fbf9', '\U0001fbf9' if self.old else
|
||
['\U0001fbf9', '\U0001fbf9', '9', '9'])
|
||
# New in 14.0.0
|
||
check('\U000107ba', '\U000107ba' if self.old else
|
||
['\U000107ba', '\U000107ba', '\U0001df1e', '\U0001df1e'])
|
||
# New in 15.0.0
|
||
check('\U0001e06d', '\U0001e06d' if self.old else
|
||
['\U0001e06d', '\U0001e06d', '\u04b1', '\u04b1'])
|
||
# New in 16.0.0
|
||
check('\U0001ccd6', '\U0001ccd6' if self.old else
|
||
['\U0001ccd6', '\U0001ccd6', 'A', 'A'])
|
||
|
||
self.assertRaises(TypeError, self.db.normalize)
|
||
self.assertRaises(TypeError, self.db.normalize, 'NFC')
|
||
self.assertRaises(ValueError, self.db.normalize, 'SPAM', 'A')
|
||
|
||
self.assertRaises(TypeError, self.db.is_normalized)
|
||
self.assertRaises(TypeError, self.db.is_normalized, 'NFC')
|
||
self.assertRaises(ValueError, self.db.is_normalized, 'SPAM', 'A')
|
||
|
||
def test_pr29(self):
|
||
# https://www.unicode.org/review/pr-29.html
|
||
# See issues #1054943 and #10254.
|
||
composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
|
||
'Li\u030dt-s\u1e73\u0301',
|
||
'\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
|
||
+ '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
|
||
'\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
|
||
+ '\u0938\u094d\u0924\u093e\u0928')
|
||
for text in composed:
|
||
self.assertEqual(self.db.normalize('NFC', text), text)
|
||
|
||
def test_issue10254(self):
|
||
# Crash reported in #10254
|
||
# New in 4.1.0
|
||
a = 'C\u0338' * 20 + 'C\u0327'
|
||
b = 'C\u0338' * 20 + '\xC7'
|
||
self.assertEqual(self.db.normalize('NFC', a), b)
|
||
|
||
def test_long_combining_mark_run(self):
|
||
# gh-149079: avoid quadratic canonical ordering.
|
||
payload = "a" + ("\u0300\u0327" * 32)
|
||
nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32)
|
||
nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31)
|
||
|
||
self.assertEqual(self.db.normalize("NFD", payload), nfd)
|
||
self.assertEqual(self.db.normalize("NFKD", payload), nfd)
|
||
self.assertEqual(self.db.normalize("NFC", payload), nfc)
|
||
self.assertEqual(self.db.normalize("NFKC", payload), nfc)
|
||
|
||
def test_combining_mark_run_fast_paths(self):
|
||
# gh-149079: cover short runs and already-sorted long runs.
|
||
short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300"
|
||
short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10)
|
||
short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9)
|
||
long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30)
|
||
long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29)
|
||
|
||
self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd)
|
||
self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd)
|
||
self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc)
|
||
self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc)
|
||
self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted)
|
||
self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted)
|
||
self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc)
|
||
self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc)
|
||
|
||
def test_issue29456(self):
|
||
# Fix #29456
|
||
u1176_str_a = '\u1100\u1176\u11a8'
|
||
u1176_str_b = '\u1100\u1176\u11a8'
|
||
u11a7_str_a = '\u1100\u1175\u11a7'
|
||
u11a7_str_b = '\uae30\u11a7'
|
||
u11c3_str_a = '\u1100\u1175\u11c3'
|
||
u11c3_str_b = '\uae30\u11c3'
|
||
self.assertEqual(self.db.normalize('NFC', u1176_str_a), u1176_str_b)
|
||
# New in 4.1.0
|
||
self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
|
||
self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
|
||
|
||
def test_east_asian_width(self):
|
||
eaw = self.db.east_asian_width
|
||
self.assertRaises(TypeError, eaw, b'a')
|
||
self.assertRaises(TypeError, eaw, bytearray())
|
||
self.assertRaises(TypeError, eaw, '')
|
||
self.assertRaises(TypeError, eaw, 'ra')
|
||
self.assertEqual(eaw('\x1e'), 'N')
|
||
self.assertEqual(eaw('\x20'), 'Na')
|
||
self.assertEqual(eaw('\uC894'), 'W')
|
||
self.assertEqual(eaw('\uFF66'), 'H')
|
||
self.assertEqual(eaw('\uFF1F'), 'F')
|
||
self.assertEqual(eaw('\u2010'), 'A')
|
||
self.assertEqual(eaw('\U00020000'), 'W')
|
||
|
||
# New in 4.1.0
|
||
self.assertEqual(eaw('\u0350'), 'N' if self.old else 'A')
|
||
self.assertEqual(eaw('\U000e01ef'), 'N' if self.old else 'A')
|
||
# New in 5.2.0
|
||
self.assertEqual(eaw('\u115a'), 'N' if self.old else 'W')
|
||
# New in 9.0.0
|
||
self.assertEqual(eaw('\u231a'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\u2614'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0001f19a'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0001f991'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0001f9c0'), 'N' if self.old else 'W')
|
||
# New in 12.0.0
|
||
self.assertEqual(eaw('\u32ff'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0001fa95'), 'N' if self.old else 'W')
|
||
# New in 13.0.0
|
||
self.assertEqual(eaw('\u31bb'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0003134a'), 'N' if self.old else 'W')
|
||
# New in 14.0.0
|
||
self.assertEqual(eaw('\u9ffd'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0002b738'), 'N' if self.old else 'W')
|
||
# New in 15.0.0
|
||
self.assertEqual(eaw('\U000323af'), 'N' if self.old else 'W')
|
||
# New in 16.0.0
|
||
self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W')
|
||
self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W')
|
||
# New in 17.0.0
|
||
self.assertEqual(eaw('\U00016FF2'), 'N' if self.old else 'W')
|
||
|
||
@unittest.skip("TODO: RUSTPYTHON; AssertionError: 'N' != 'W' (passed on latest, fails on 3.2)")
|
||
def test_east_asian_width_unassigned(self):
|
||
eaw = self.db.east_asian_width
|
||
# unassigned
|
||
for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2':
|
||
self.assertEqual(eaw(char), 'N')
|
||
self.assertIs(self.db.name(char, None), None)
|
||
|
||
# unassigned but reserved for CJK
|
||
for char in ('\U0002A6E0\U0002FA20\U0003134B\U0003FFFD'
|
||
'\uFA6E\uFADA'): # New in 5.2.0
|
||
self.assertEqual(eaw(char), 'W')
|
||
self.assertIs(self.db.name(char, None), None)
|
||
|
||
# private use areas
|
||
for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0':
|
||
self.assertEqual(eaw(char), 'A')
|
||
self.assertIs(self.db.name(char, None), None)
|
||
|
||
class UnicodeFunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
|
||
db = unicodedata
|
||
old = False
|
||
|
||
# Update this if the database changes. Make sure to do a full rebuild
|
||
# (e.g. 'make distclean && make') to get the correct checksum.
|
||
expectedchecksum = ('00b13fa975a60b1d3f490f1fc8c126ab24990c75'
|
||
if quicktest else
|
||
'ebfc9dd281c2226998fd435744dd2e9321899beb')
|
||
|
||
@requires_resource('network')
|
||
def test_all_names(self):
|
||
TESTDATAFILE = "DerivedName.txt"
|
||
testdata = download_test_data_file(TESTDATAFILE)
|
||
|
||
with testdata:
|
||
self.run_name_tests(testdata)
|
||
|
||
def run_name_tests(self, testdata):
|
||
names_ref = {}
|
||
|
||
def parse_cp(s):
|
||
return int(s, 16)
|
||
|
||
# Parse data
|
||
for line in testdata:
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
raw_cp, name = line.split("; ")
|
||
# Check for a range
|
||
if ".." in raw_cp:
|
||
cp1, cp2 = map(parse_cp, raw_cp.split(".."))
|
||
# remove ‘*’ at the end
|
||
assert name[-1] == '*', (raw_cp, name)
|
||
name = name[:-1]
|
||
for cp in range(cp1, cp2 + 1):
|
||
names_ref[cp] = f"{name}{cp:04X}"
|
||
elif name[-1] == '*':
|
||
cp = parse_cp(raw_cp)
|
||
name = name[:-1]
|
||
names_ref[cp] = f"{name}{cp:04X}"
|
||
else:
|
||
assert '*' not in name, (raw_cp, name)
|
||
cp = parse_cp(raw_cp)
|
||
names_ref[cp] = name
|
||
|
||
for cp in range(0, sys.maxunicode + 1):
|
||
self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
|
||
def test_isxidstart(self):
|
||
self.assertTrue(self.db.isxidstart('S'))
|
||
self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM
|
||
self.assertTrue(self.db.isxidstart('\u0EC6')) # LAO KO LA
|
||
self.assertTrue(self.db.isxidstart('\u17DC')) # KHMER SIGN AVAKRAHASANYA
|
||
self.assertTrue(self.db.isxidstart('\uA015')) # YI SYLLABLE WU
|
||
self.assertTrue(self.db.isxidstart('\uFE7B')) # ARABIC KASRA MEDIAL FORM
|
||
|
||
self.assertFalse(self.db.isxidstart(' '))
|
||
self.assertFalse(self.db.isxidstart('0'))
|
||
self.assertRaises(TypeError, self.db.isxidstart)
|
||
self.assertRaises(TypeError, self.db.isxidstart, 'xx')
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
|
||
def test_isxidcontinue(self):
|
||
self.assertTrue(self.db.isxidcontinue('S'))
|
||
self.assertTrue(self.db.isxidcontinue('_'))
|
||
self.assertTrue(self.db.isxidcontinue('0'))
|
||
self.assertTrue(self.db.isxidcontinue('\u00BA')) # MASCULINE ORDINAL INDICATOR
|
||
self.assertTrue(self.db.isxidcontinue('\u0640')) # ARABIC TATWEEL
|
||
self.assertTrue(self.db.isxidcontinue('\u0710')) # SYRIAC LETTER ALAPH
|
||
self.assertTrue(self.db.isxidcontinue('\u0B3E')) # ORIYA VOWEL SIGN AA
|
||
self.assertTrue(self.db.isxidcontinue('\u17D7')) # KHMER SIGN LEK TOO
|
||
|
||
self.assertFalse(self.db.isxidcontinue(' '))
|
||
self.assertRaises(TypeError, self.db.isxidcontinue)
|
||
self.assertRaises(TypeError, self.db.isxidcontinue, 'xx')
|
||
|
||
def test_grapheme_cluster_break(self):
|
||
gcb = self.db.grapheme_cluster_break
|
||
self.assertEqual(gcb(' '), 'Other')
|
||
self.assertEqual(gcb('x'), 'Other')
|
||
self.assertEqual(gcb('\U0010FFFF'), 'Other')
|
||
self.assertEqual(gcb('\r'), 'CR')
|
||
self.assertEqual(gcb('\n'), 'LF')
|
||
self.assertEqual(gcb('\0'), 'Control')
|
||
self.assertEqual(gcb('\t'), 'Control')
|
||
self.assertEqual(gcb('\x1F'), 'Control')
|
||
self.assertEqual(gcb('\x7F'), 'Control')
|
||
self.assertEqual(gcb('\x9F'), 'Control')
|
||
self.assertEqual(gcb('\U000E0001'), 'Control')
|
||
self.assertEqual(gcb('\u0300'), 'Extend')
|
||
self.assertEqual(gcb('\u200C'), 'Extend')
|
||
self.assertEqual(gcb('\U000E01EF'), 'Extend')
|
||
self.assertEqual(gcb('\u1159'), 'L')
|
||
self.assertEqual(gcb('\u11F9'), 'T')
|
||
self.assertEqual(gcb('\uD788'), 'LV')
|
||
self.assertEqual(gcb('\uD7A3'), 'LVT')
|
||
# New in 5.0.0
|
||
self.assertEqual(gcb('\u05BA'), 'Extend')
|
||
self.assertEqual(gcb('\u20EF'), 'Extend')
|
||
# New in 5.1.0
|
||
self.assertEqual(gcb('\u2064'), 'Control')
|
||
self.assertEqual(gcb('\uAA4D'), 'SpacingMark')
|
||
# New in 5.2.0
|
||
self.assertEqual(gcb('\u0816'), 'Extend')
|
||
self.assertEqual(gcb('\uA97C'), 'L')
|
||
self.assertEqual(gcb('\uD7C6'), 'V')
|
||
self.assertEqual(gcb('\uD7FB'), 'T')
|
||
# New in 6.0.0
|
||
self.assertEqual(gcb('\u093A'), 'Extend')
|
||
self.assertEqual(gcb('\U00011002'), 'SpacingMark')
|
||
# New in 6.1.0
|
||
self.assertEqual(gcb('\U000E0FFF'), 'Control')
|
||
self.assertEqual(gcb('\U00016F7E'), 'SpacingMark')
|
||
# New in 6.2.0
|
||
self.assertEqual(gcb('\U0001F1E6'), 'Regional_Indicator')
|
||
self.assertEqual(gcb('\U0001F1FF'), 'Regional_Indicator')
|
||
# New in 6.3.0
|
||
self.assertEqual(gcb('\u180E'), 'Control')
|
||
self.assertEqual(gcb('\u1A1B'), 'Extend')
|
||
# New in 7.0.0
|
||
self.assertEqual(gcb('\u0E33'), 'SpacingMark')
|
||
self.assertEqual(gcb('\u0EB3'), 'SpacingMark')
|
||
self.assertEqual(gcb('\U0001BCA3'), 'Control')
|
||
self.assertEqual(gcb('\U0001E8D6'), 'Extend')
|
||
self.assertEqual(gcb('\U0001163E'), 'SpacingMark')
|
||
# New in 8.0.0
|
||
self.assertEqual(gcb('\u08E3'), 'Extend')
|
||
self.assertEqual(gcb('\U00011726'), 'SpacingMark')
|
||
# New in 9.0.0
|
||
self.assertEqual(gcb('\u0600'), 'Prepend')
|
||
self.assertEqual(gcb('\U000E007F'), 'Extend')
|
||
self.assertEqual(gcb('\U00011CB4'), 'SpacingMark')
|
||
self.assertEqual(gcb('\u200D'), 'ZWJ')
|
||
# New in 10.0.0
|
||
self.assertEqual(gcb('\U00011D46'), 'Prepend')
|
||
self.assertEqual(gcb('\U00011D47'), 'Extend')
|
||
self.assertEqual(gcb('\U00011A97'), 'SpacingMark')
|
||
# New in 11.0.0
|
||
self.assertEqual(gcb('\U000110CD'), 'Prepend')
|
||
self.assertEqual(gcb('\u07FD'), 'Extend')
|
||
self.assertEqual(gcb('\U00011EF6'), 'SpacingMark')
|
||
# New in 12.0.0
|
||
self.assertEqual(gcb('\U00011A84'), 'Prepend')
|
||
self.assertEqual(gcb('\U00013438'), 'Control')
|
||
self.assertEqual(gcb('\U0001E2EF'), 'Extend')
|
||
self.assertEqual(gcb('\U00016F87'), 'SpacingMark')
|
||
# New in 13.0.0
|
||
self.assertEqual(gcb('\U00011941'), 'Prepend')
|
||
self.assertEqual(gcb('\U00016FE4'), 'Extend')
|
||
self.assertEqual(gcb('\U00011942'), 'SpacingMark')
|
||
# New in 14.0.0
|
||
self.assertEqual(gcb('\u0891'), 'Prepend')
|
||
self.assertEqual(gcb('\U0001E2AE'), 'Extend')
|
||
# New in 15.0.0
|
||
self.assertEqual(gcb('\U00011F02'), 'Prepend')
|
||
self.assertEqual(gcb('\U0001343F'), 'Control')
|
||
self.assertEqual(gcb('\U0001E4EF'), 'Extend')
|
||
self.assertEqual(gcb('\U00011F3F'), 'SpacingMark')
|
||
# New in 16.0.0
|
||
self.assertEqual(gcb('\U000113D1'), 'Prepend')
|
||
self.assertEqual(gcb('\U0001E5EF'), 'Extend')
|
||
self.assertEqual(gcb('\U0001612C'), 'SpacingMark')
|
||
self.assertEqual(gcb('\U00016D63'), 'V')
|
||
# New in 17.0.0
|
||
self.assertEqual(gcb('\u1AEB'), 'Extend')
|
||
self.assertEqual(gcb('\U00011B67'), 'SpacingMark')
|
||
|
||
self.assertRaises(TypeError, gcb)
|
||
self.assertRaises(TypeError, gcb, b'x')
|
||
self.assertRaises(TypeError, gcb, 120)
|
||
self.assertRaises(TypeError, gcb, '')
|
||
self.assertRaises(TypeError, gcb, 'xx')
|
||
|
||
def test_indic_conjunct_break(self):
|
||
incb = self.db.indic_conjunct_break
|
||
self.assertEqual(incb(' '), 'None')
|
||
self.assertEqual(incb('x'), 'None')
|
||
self.assertEqual(incb('\U0010FFFF'), 'None')
|
||
# New in 15.1.0
|
||
self.assertEqual(incb('\u094D'), 'Linker')
|
||
self.assertEqual(incb('\u0D4D'), 'Linker')
|
||
self.assertEqual(incb('\u0915'), 'Consonant')
|
||
self.assertEqual(incb('\u0D3A'), 'Consonant')
|
||
self.assertEqual(incb('\u0300'), 'Extend')
|
||
self.assertEqual(incb('\U0001E94A'), 'Extend')
|
||
# New in 16.0.0
|
||
self.assertEqual(incb('\u034F'), 'Extend')
|
||
self.assertEqual(incb('\U000E01EF'), 'Extend')
|
||
# New in 17.0.0
|
||
self.assertEqual(incb('\u1039'), 'Linker')
|
||
self.assertEqual(incb('\U00011F42'), 'Linker')
|
||
self.assertEqual(incb('\u1000'), 'Consonant')
|
||
self.assertEqual(incb('\U00011F33'), 'Consonant')
|
||
self.assertEqual(incb('\U0001E6F5'), 'Extend')
|
||
|
||
self.assertRaises(TypeError, incb)
|
||
self.assertRaises(TypeError, incb, b'x')
|
||
self.assertRaises(TypeError, incb, 120)
|
||
self.assertRaises(TypeError, incb, '')
|
||
self.assertRaises(TypeError, incb, 'xx')
|
||
|
||
def test_extended_pictographic(self):
|
||
ext_pict = self.db.extended_pictographic
|
||
self.assertIs(ext_pict(' '), False)
|
||
self.assertIs(ext_pict('x'), False)
|
||
self.assertIs(ext_pict('\U0010FFFF'), False)
|
||
# New in 13.0.0
|
||
self.assertIs(ext_pict('\xA9'), True)
|
||
self.assertIs(ext_pict('\u203C'), True)
|
||
self.assertIs(ext_pict('\U0001FAD6'), True)
|
||
self.assertIs(ext_pict('\U0001FFFD'), True)
|
||
# New in 17.0.0
|
||
self.assertIs(ext_pict('\u2388'), False)
|
||
self.assertIs(ext_pict('\U0001FA6D'), False)
|
||
|
||
self.assertRaises(TypeError, ext_pict)
|
||
self.assertRaises(TypeError, ext_pict, b'x')
|
||
self.assertRaises(TypeError, ext_pict, 120)
|
||
self.assertRaises(TypeError, ext_pict, '')
|
||
self.assertRaises(TypeError, ext_pict, 'xx')
|
||
|
||
def test_grapheme_break(self):
|
||
def graphemes(*args):
|
||
return list(map(str, self.db.iter_graphemes(*args)))
|
||
|
||
self.assertRaises(TypeError, self.db.iter_graphemes)
|
||
self.assertRaises(TypeError, self.db.iter_graphemes, b'x')
|
||
self.assertRaises(TypeError, self.db.iter_graphemes, 'x', 0, 0, 0)
|
||
|
||
self.assertEqual(graphemes(''), [])
|
||
self.assertEqual(graphemes('abcd'), ['a', 'b', 'c', 'd'])
|
||
self.assertEqual(graphemes('abcd', 1), ['b', 'c', 'd'])
|
||
self.assertEqual(graphemes('abcd', 1, 3), ['b', 'c'])
|
||
self.assertEqual(graphemes('abcd', -3), ['b', 'c', 'd'])
|
||
self.assertEqual(graphemes('abcd', 1, -1), ['b', 'c'])
|
||
self.assertEqual(graphemes('abcd', 3, 1), [])
|
||
self.assertEqual(graphemes('abcd', 5), [])
|
||
self.assertEqual(graphemes('abcd', 0, 5), ['a', 'b', 'c', 'd'])
|
||
self.assertEqual(graphemes('abcd', -5), ['a', 'b', 'c', 'd'])
|
||
self.assertEqual(graphemes('abcd', 0, -5), [])
|
||
# GB3
|
||
self.assertEqual(graphemes('\r\n'), ['\r\n'])
|
||
# GB4
|
||
self.assertEqual(graphemes('\r\u0308'), ['\r', '\u0308'])
|
||
self.assertEqual(graphemes('\n\u0308'), ['\n', '\u0308'])
|
||
self.assertEqual(graphemes('\0\u0308'), ['\0', '\u0308'])
|
||
# GB5
|
||
self.assertEqual(graphemes('\u06dd\r'), ['\u06dd', '\r'])
|
||
self.assertEqual(graphemes('\u06dd\n'), ['\u06dd', '\n'])
|
||
self.assertEqual(graphemes('\u06dd\0'), ['\u06dd', '\0'])
|
||
# GB6
|
||
self.assertEqual(graphemes('\u1100\u1160'), ['\u1100\u1160'])
|
||
self.assertEqual(graphemes('\u1100\uAC00'), ['\u1100\uAC00'])
|
||
self.assertEqual(graphemes('\u1100\uAC01'), ['\u1100\uAC01'])
|
||
# GB7
|
||
self.assertEqual(graphemes('\uAC00\u1160'), ['\uAC00\u1160'])
|
||
self.assertEqual(graphemes('\uAC00\u11A8'), ['\uAC00\u11A8'])
|
||
self.assertEqual(graphemes('\u1160\u1160'), ['\u1160\u1160'])
|
||
self.assertEqual(graphemes('\u1160\u11A8'), ['\u1160\u11A8'])
|
||
# GB8
|
||
self.assertEqual(graphemes('\uAC01\u11A8'), ['\uAC01\u11A8'])
|
||
self.assertEqual(graphemes('\u11A8\u11A8'), ['\u11A8\u11A8'])
|
||
# GB9
|
||
self.assertEqual(graphemes('a\u0300'), ['a\u0300'])
|
||
self.assertEqual(graphemes('a\u200D'), ['a\u200D'])
|
||
# GB9a
|
||
self.assertEqual(graphemes('\u0905\u0903'), ['\u0905\u0903'])
|
||
# GB9b
|
||
self.assertEqual(graphemes('\u06dd\u0661'), ['\u06dd\u0661'])
|
||
# GB9c
|
||
self.assertEqual(graphemes('\u0915\u094d\u0924'),
|
||
['\u0915\u094d\u0924'])
|
||
self.assertEqual(graphemes('\u0915\u094D\u094D\u0924'),
|
||
['\u0915\u094D\u094D\u0924'])
|
||
self.assertEqual(graphemes('\u0915\u094D\u0924\u094D\u092F'),
|
||
['\u0915\u094D\u0924\u094D\u092F'])
|
||
# GB11
|
||
self.assertEqual(graphemes(
|
||
'\U0001F9D1\U0001F3FE\u200D\u2764\uFE0F'
|
||
'\u200D\U0001F48B\u200D\U0001F9D1\U0001F3FC'),
|
||
['\U0001F9D1\U0001F3FE\u200D\u2764\uFE0F'
|
||
'\u200D\U0001F48B\u200D\U0001F9D1\U0001F3FC'])
|
||
# GB12
|
||
self.assertEqual(graphemes(
|
||
'\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
|
||
['\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])
|
||
# GB13
|
||
self.assertEqual(graphemes(
|
||
'a\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
|
||
['a', '\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
|
||
def test_block(self):
|
||
self.assertEqual(self.db.block('\u0000'), 'Basic Latin')
|
||
self.assertEqual(self.db.block('\u0041'), 'Basic Latin')
|
||
self.assertEqual(self.db.block('\u007F'), 'Basic Latin')
|
||
self.assertEqual(self.db.block('\u0080'), 'Latin-1 Supplement')
|
||
self.assertEqual(self.db.block('\u00FF'), 'Latin-1 Supplement')
|
||
self.assertEqual(self.db.block('\u1159'), 'Hangul Jamo')
|
||
self.assertEqual(self.db.block('\u11F9'), 'Hangul Jamo')
|
||
self.assertEqual(self.db.block('\uD788'), 'Hangul Syllables')
|
||
self.assertEqual(self.db.block('\uD7A3'), 'Hangul Syllables')
|
||
# New in 5.0.0
|
||
self.assertEqual(self.db.block('\u05BA'), 'Hebrew')
|
||
self.assertEqual(self.db.block('\u20EF'), 'Combining Diacritical Marks for Symbols')
|
||
# New in 5.1.0
|
||
self.assertEqual(self.db.block('\u2064'), 'General Punctuation')
|
||
self.assertEqual(self.db.block('\uAA4D'), 'Cham')
|
||
# New in 5.2.0
|
||
self.assertEqual(self.db.block('\u0816'), 'Samaritan')
|
||
self.assertEqual(self.db.block('\uA97C'), 'Hangul Jamo Extended-A')
|
||
self.assertEqual(self.db.block('\uD7C6'), 'Hangul Jamo Extended-B')
|
||
self.assertEqual(self.db.block('\uD7FB'), 'Hangul Jamo Extended-B')
|
||
# New in 6.0.0
|
||
self.assertEqual(self.db.block('\u093A'), 'Devanagari')
|
||
self.assertEqual(self.db.block('\U00011002'), 'Brahmi')
|
||
# New in 6.1.0
|
||
self.assertEqual(self.db.block('\U000E0FFF'), 'No_Block')
|
||
self.assertEqual(self.db.block('\U00016F7E'), 'Miao')
|
||
# New in 6.2.0
|
||
self.assertEqual(self.db.block('\U0001F1E6'), 'Enclosed Alphanumeric Supplement')
|
||
self.assertEqual(self.db.block('\U0001F1FF'), 'Enclosed Alphanumeric Supplement')
|
||
# New in 6.3.0
|
||
self.assertEqual(self.db.block('\u180E'), 'Mongolian')
|
||
self.assertEqual(self.db.block('\u1A1B'), 'Buginese')
|
||
# New in 7.0.0
|
||
self.assertEqual(self.db.block('\u0E33'), 'Thai')
|
||
self.assertEqual(self.db.block('\u0EB3'), 'Lao')
|
||
self.assertEqual(self.db.block('\U0001BCA3'), 'Shorthand Format Controls')
|
||
self.assertEqual(self.db.block('\U0001E8D6'), 'Mende Kikakui')
|
||
self.assertEqual(self.db.block('\U0001163E'), 'Modi')
|
||
# New in 8.0.0
|
||
self.assertEqual(self.db.block('\u08E3'), 'Arabic Extended-A')
|
||
self.assertEqual(self.db.block('\U00011726'), 'Ahom')
|
||
# New in 9.0.0
|
||
self.assertEqual(self.db.block('\u0600'), 'Arabic')
|
||
self.assertEqual(self.db.block('\U000E007F'), 'Tags')
|
||
self.assertEqual(self.db.block('\U00011CB4'), 'Marchen')
|
||
self.assertEqual(self.db.block('\u200D'), 'General Punctuation')
|
||
# New in 10.0.0
|
||
self.assertEqual(self.db.block('\U00011D46'), 'Masaram Gondi')
|
||
self.assertEqual(self.db.block('\U00011D47'), 'Masaram Gondi')
|
||
self.assertEqual(self.db.block('\U00011A97'), 'Soyombo')
|
||
# New in 11.0.0
|
||
self.assertEqual(self.db.block('\U000110CD'), 'Kaithi')
|
||
self.assertEqual(self.db.block('\u07FD'), 'NKo')
|
||
self.assertEqual(self.db.block('\U00011EF6'), 'Makasar')
|
||
# New in 12.0.0
|
||
self.assertEqual(self.db.block('\U00011A84'), 'Soyombo')
|
||
self.assertEqual(self.db.block('\U00013438'), 'Egyptian Hieroglyph Format Controls')
|
||
self.assertEqual(self.db.block('\U0001E2EF'), 'Wancho')
|
||
self.assertEqual(self.db.block('\U00016F87'), 'Miao')
|
||
# New in 13.0.0
|
||
self.assertEqual(self.db.block('\U00011941'), 'Dives Akuru')
|
||
self.assertEqual(self.db.block('\U00016FE4'), 'Ideographic Symbols and Punctuation')
|
||
self.assertEqual(self.db.block('\U00011942'), 'Dives Akuru')
|
||
# New in 14.0.0
|
||
self.assertEqual(self.db.block('\u0891'), 'Arabic Extended-B')
|
||
self.assertEqual(self.db.block('\U0001E2AE'), 'Toto')
|
||
# New in 15.0.0
|
||
self.assertEqual(self.db.block('\U00011F02'), 'Kawi')
|
||
self.assertEqual(self.db.block('\U0001343F'), 'Egyptian Hieroglyph Format Controls')
|
||
self.assertEqual(self.db.block('\U0001E4EF'), 'Nag Mundari')
|
||
self.assertEqual(self.db.block('\U00011F3F'), 'Kawi')
|
||
# New in 16.0.0
|
||
self.assertEqual(self.db.block('\U000113D1'), 'Tulu-Tigalari')
|
||
self.assertEqual(self.db.block('\U0001E5EF'), 'Ol Onal')
|
||
self.assertEqual(self.db.block('\U0001612C'), 'Gurung Khema')
|
||
self.assertEqual(self.db.block('\U00016D63'), 'Kirat Rai')
|
||
# New in 17.0.0
|
||
self.assertEqual(self.db.block('\u1AEB'), 'Combining Diacritical Marks Extended')
|
||
self.assertEqual(self.db.block('\U00011B67'), 'Sharada Supplement')
|
||
# Unassigned
|
||
self.assertEqual(self.db.block('\U00100000'), 'Supplementary Private Use Area-B')
|
||
self.assertEqual(self.db.block('\U0010FFFF'), 'Supplementary Private Use Area-B')
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
|
||
def test_block_invalid_input(self):
|
||
self.assertRaises(TypeError, self.db.block)
|
||
self.assertRaises(TypeError, self.db.block, b'x')
|
||
self.assertRaises(TypeError, self.db.block, 120)
|
||
self.assertRaises(TypeError, self.db.block, '')
|
||
self.assertRaises(TypeError, self.db.block, 'xx')
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; + N
|
||
def test_east_asian_width_9_0_changes(self):
|
||
return super().test_east_asian_width_9_0_changes()
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'D4CC 11B6' != '1111 1171 11B6'
|
||
def test_decomposition(self):
|
||
return super().test_decomposition()
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
|
||
def test_lookup_nonexistant(self):
|
||
return super().test_lookup_nonexistant()
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'digit'
|
||
def test_function_checksum(self):
|
||
return super().test_function_checksum()
|
||
|
||
|
||
class Unicode_3_2_0_FunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
|
||
db = unicodedata.ucd_3_2_0
|
||
old = True
|
||
expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47'
|
||
if quicktest else
|
||
'68cd01e2c680b851c1fcab012efb5635b2229c2b')
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON
|
||
def test_normalization(self):
|
||
return super().test_normalization()
|
||
|
||
@unittest.expectedSuccess # TODO: RUSTPYTHON
|
||
def test_combining(self):
|
||
return super().test_combining()
|
||
|
||
|
||
class UnicodeMiscTest(unittest.TestCase):
|
||
db = unicodedata
|
||
|
||
@cpython_only
|
||
def test_disallow_instantiation(self):
|
||
# Ensure that the type disallows instantiation (bpo-43916)
|
||
check_disallow_instantiation(self, unicodedata.UCD)
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; ---
|
||
@force_not_colorized
|
||
def test_failed_import_during_compiling(self):
|
||
# Issue 4367
|
||
# Decoding \N escapes requires the unicodedata module. If it can't be
|
||
# imported, we shouldn't segfault.
|
||
|
||
# This program should raise a SyntaxError in the eval.
|
||
code = "import sys;" \
|
||
"sys.modules['unicodedata'] = None;" \
|
||
"""eval("'\\\\N{SOFT HYPHEN}'")"""
|
||
# We use a separate process because the unicodedata module may already
|
||
# have been loaded in this process.
|
||
result = script_helper.assert_python_failure("-c", code)
|
||
error = "SyntaxError: (unicode error) \\N escapes not supported " \
|
||
"(can't load unicodedata module)"
|
||
self.assertIn(error, result.err.decode("ascii"))
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError Process return code is 1
|
||
def test_unicodedata_unload_reload(self):
|
||
# gh-149449: dropping unicodedata and running gc must not leave the
|
||
# cached _ucnhash_CAPI pointer dangling.
|
||
code = (
|
||
"import gc, sys\n"
|
||
"assert '\\N{GRINNING FACE}'.encode("
|
||
" 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n"
|
||
"compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '<x>', 'exec')\n"
|
||
"del sys.modules['unicodedata']\n"
|
||
"gc.collect()\n"
|
||
"assert '\\N{WINKING FACE}'.encode("
|
||
" 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n"
|
||
"compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '<x>', 'exec')\n"
|
||
)
|
||
script_helper.assert_python_ok("-c", code)
|
||
|
||
def test_decimal_numeric_consistent(self):
|
||
# Test that decimal and numeric are consistent,
|
||
# i.e. if a character has a decimal value,
|
||
# its numeric value should be the same.
|
||
count = 0
|
||
for c in iterallchars():
|
||
dec = self.db.decimal(c, -1)
|
||
if dec != -1:
|
||
self.assertEqual(dec, self.db.numeric(c))
|
||
count += 1
|
||
self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
|
||
|
||
def test_digit_numeric_consistent(self):
|
||
# Test that digit and numeric are consistent,
|
||
# i.e. if a character has a digit value,
|
||
# its numeric value should be the same.
|
||
count = 0
|
||
for c in iterallchars():
|
||
dec = self.db.digit(c, -1)
|
||
if dec != -1:
|
||
self.assertEqual(dec, self.db.numeric(c))
|
||
count += 1
|
||
self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
|
||
|
||
def test_normalize_consistent(self):
|
||
allchars = list(iterallchars())
|
||
for form in ('NFC', 'NFD', 'NFKC', 'NFKD'):
|
||
for c in allchars:
|
||
norm = self.db.normalize(form, c)
|
||
self.assertEqual(self.db.is_normalized(form, c), norm == c)
|
||
if norm != c:
|
||
self.assertEqual(self.db.normalize(form, norm), norm)
|
||
self.assertTrue(self.db.is_normalized(form, norm))
|
||
|
||
def test_bug_1704793(self):
|
||
self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
|
||
|
||
def test_ucd_510(self):
|
||
# In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
|
||
self.assertTrue(unicodedata.mirrored("\u0f3a"))
|
||
self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
|
||
# Also, we now have two ways of representing
|
||
# the upper-case mapping: as delta, or as absolute value
|
||
self.assertTrue("a".upper()=='A')
|
||
self.assertTrue("\u1d79".upper()=='\ua77d')
|
||
self.assertTrue(".".upper()=='.')
|
||
|
||
@requires_resource('cpu')
|
||
def test_bug_5828(self):
|
||
self.assertEqual("\u1d79".lower(), "\u1d79")
|
||
# Only U+0000 should have U+0000 as its upper/lower/titlecase variant
|
||
self.assertEqual(
|
||
[
|
||
c for c in iterallchars()
|
||
if "\x00" in (c.lower(), c.upper(), c.title())
|
||
],
|
||
["\x00"]
|
||
)
|
||
|
||
def test_bug_4971(self):
|
||
# LETTER DZ WITH CARON: DZ, Dz, dz
|
||
self.assertEqual("\u01c4".title(), "\u01c5")
|
||
self.assertEqual("\u01c5".title(), "\u01c5")
|
||
self.assertEqual("\u01c6".title(), "\u01c5")
|
||
|
||
def test_linebreak_7643(self):
|
||
for c in iterallchars():
|
||
lines = (c + 'A').splitlines()
|
||
if c in ('\x0a', '\x0b', '\x0c', '\x0d', '\x85',
|
||
'\x1c', '\x1d', '\x1e', '\u2028', '\u2029'):
|
||
self.assertEqual(len(lines), 2,
|
||
r"%a should be a linebreak" % c)
|
||
else:
|
||
self.assertEqual(len(lines), 1,
|
||
r"%a should not be a linebreak" % c)
|
||
|
||
def test_segment_object(self):
|
||
segments = list(unicodedata.iter_graphemes('spa\u0300m'))
|
||
self.assertEqual(len(segments), 4, segments)
|
||
segment = segments[2]
|
||
self.assertEqual(segment.start, 2)
|
||
self.assertEqual(segment.end, 4)
|
||
self.assertEqual(str(segment), 'a\u0300')
|
||
self.assertEqual(repr(segment), '<Segment 2:4>')
|
||
self.assertRaises(TypeError, iter, segment)
|
||
self.assertRaises(TypeError, len, segment)
|
||
|
||
|
||
class NormalizationTest(unittest.TestCase):
|
||
@staticmethod
|
||
def unistr(data):
|
||
data = [int(x, 16) for x in data.split(" ")]
|
||
return "".join([chr(x) for x in data])
|
||
|
||
@requires_resource('network')
|
||
@requires_resource('cpu')
|
||
def test_normalization(self):
|
||
TESTDATAFILE = "NormalizationTest.txt"
|
||
testdata = download_test_data_file(TESTDATAFILE)
|
||
|
||
with testdata:
|
||
self.run_normalization_tests(testdata, unicodedata)
|
||
|
||
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : 2F868;2136A;2136A;2136A;2136A
|
||
@requires_resource('cpu')
|
||
def test_normalization_3_2_0(self):
|
||
testdatafile = findfile('NormalizationTest-3.2.0.txt')
|
||
with open(testdatafile, encoding='utf-8') as testdata:
|
||
self.run_normalization_tests(testdata, unicodedata.ucd_3_2_0)
|
||
|
||
def run_normalization_tests(self, testdata, ucd):
|
||
part = None
|
||
part1_data = set()
|
||
|
||
NFC = partial(ucd.normalize, "NFC")
|
||
NFKC = partial(ucd.normalize, "NFKC")
|
||
NFD = partial(ucd.normalize, "NFD")
|
||
NFKD = partial(ucd.normalize, "NFKD")
|
||
is_normalized = ucd.is_normalized
|
||
|
||
for line in testdata:
|
||
if '#' in line:
|
||
line = line.split('#')[0]
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
if line.startswith("@Part"):
|
||
part = line.split()[0]
|
||
continue
|
||
c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]]
|
||
|
||
# Perform tests
|
||
self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
|
||
self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
|
||
self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
|
||
self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
|
||
self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
|
||
NFKC(c3) == NFKC(c4) == NFKC(c5),
|
||
line)
|
||
self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
|
||
NFKD(c3) == NFKD(c4) == NFKD(c5),
|
||
line)
|
||
|
||
self.assertTrue(is_normalized("NFC", c2))
|
||
self.assertTrue(is_normalized("NFC", c4))
|
||
|
||
self.assertTrue(is_normalized("NFD", c3))
|
||
self.assertTrue(is_normalized("NFD", c5))
|
||
|
||
self.assertTrue(is_normalized("NFKC", c4))
|
||
self.assertTrue(is_normalized("NFKD", c5))
|
||
|
||
# Record part 1 data
|
||
if part == "@Part1":
|
||
part1_data.add(c1)
|
||
|
||
# Perform tests for all other data
|
||
for X in iterallchars():
|
||
if X in part1_data:
|
||
continue
|
||
self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), ord(X))
|
||
|
||
def test_edge_cases(self):
|
||
self.assertRaises(TypeError, unicodedata.normalize)
|
||
self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx')
|
||
self.assertEqual(unicodedata.normalize('NFKC', ''), '')
|
||
|
||
def test_bug_834676(self):
|
||
# Check for bug 834676
|
||
unicodedata.normalize('NFC', '\ud55c\uae00')
|
||
|
||
def test_normalize_return_type(self):
|
||
# gh-129569: normalize() return type must always be str
|
||
normalize = unicodedata.normalize
|
||
|
||
class MyStr(str):
|
||
pass
|
||
|
||
normalization_forms = ("NFC", "NFKC", "NFD", "NFKD")
|
||
input_strings = (
|
||
# normalized strings
|
||
"",
|
||
"ascii",
|
||
# unnormalized strings
|
||
"\u1e0b\u0323",
|
||
"\u0071\u0307\u0323",
|
||
)
|
||
|
||
for form in normalization_forms:
|
||
for input_str in input_strings:
|
||
with self.subTest(form=form, input_str=input_str):
|
||
self.assertIs(type(normalize(form, input_str)), str)
|
||
self.assertIs(type(normalize(form, MyStr(input_str))), str)
|
||
|
||
|
||
class GraphemeBreakTest(unittest.TestCase):
|
||
@requires_resource('network')
|
||
def test_grapheme_break(self):
|
||
TESTDATAFILE = "GraphemeBreakTest.txt"
|
||
testdata = download_test_data_file(TESTDATAFILE)
|
||
|
||
with testdata:
|
||
self.run_grapheme_break_tests(testdata)
|
||
|
||
def run_grapheme_break_tests(self, testdata):
|
||
for line in testdata:
|
||
line, _, comment = line.partition('#')
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
comment = comment.strip()
|
||
|
||
chunks = []
|
||
breaks = []
|
||
pos = 0
|
||
for field in line.replace('×', ' ').split():
|
||
if field == '÷':
|
||
chunks.append('')
|
||
breaks.append(pos)
|
||
else:
|
||
chunks[-1] += chr(int(field, 16))
|
||
pos += 1
|
||
self.assertEqual(chunks.pop(), '', line)
|
||
input = ''.join(chunks)
|
||
with self.subTest(line):
|
||
result = list(unicodedata.iter_graphemes(input))
|
||
self.assertEqual(list(map(str, result)), chunks, comment)
|
||
self.assertEqual([x.start for x in result], breaks[:-1], comment)
|
||
self.assertEqual([x.end for x in result], breaks[1:], comment)
|
||
for i in range(1, len(breaks) - 1):
|
||
result = list(unicodedata.iter_graphemes(input, breaks[i]))
|
||
self.assertEqual(list(map(str, result)), chunks[i:], comment)
|
||
self.assertEqual([x.start for x in result], breaks[i:-1], comment)
|
||
self.assertEqual([x.end for x in result], breaks[i+1:], comment)
|
||
|
||
def test_reference_loops(self):
|
||
# Test that reference loops involving GraphemeBreakIterator or
|
||
# Segment can be broken by the garbage collector.
|
||
class S(str):
|
||
pass
|
||
|
||
s = S('abc')
|
||
s.ref = unicodedata.iter_graphemes(s)
|
||
wr = weakref.ref(s)
|
||
del s
|
||
self.assertIsNotNone(wr())
|
||
gc_collect()
|
||
self.assertIsNone(wr())
|
||
|
||
s = S('abc')
|
||
s.ref = next(unicodedata.iter_graphemes(s))
|
||
wr = weakref.ref(s)
|
||
del s
|
||
self.assertIsNotNone(wr())
|
||
gc_collect()
|
||
self.assertIsNone(wr())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|