From ff7b96f91cabb3481bac75d936e0f092a4e4d94a Mon Sep 17 00:00:00 2001 From: Jeong Yunwon Date: Mon, 18 Jul 2022 02:50:33 +0900 Subject: [PATCH] Update test_unicode* from CPython 3.10.5 --- Lib/test/test_unicode.py | 232 ++++++++++++++++++------ Lib/test/test_unicode_file_functions.py | 20 +- Lib/test/test_unicode_identifiers.py | 8 +- 3 files changed, 192 insertions(+), 68 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index bf7feb66f..8dd25fcca 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -11,11 +11,14 @@ import itertools import operator import struct import sys +import textwrap import unicodedata import unittest import warnings +from test.support import import_helper +from test.support import warnings_helper from test import support, string_tests -from test.support import import_helper, warnings_helper +from test.support.script_helper import assert_python_failure # Error handling (bad decoder return) def search_function(encoding): @@ -33,7 +36,6 @@ def search_function(encoding): return (encode2, decode2, None, None) else: return None -codecs.register(search_function) def duplicate_string(text): """ @@ -55,6 +57,10 @@ class UnicodeTest(string_tests.CommonTest, type2test = str + def setUp(self): + codecs.register(search_function) + self.addCleanup(codecs.unregister, search_function) + def checkequalnofix(self, result, object, methodname, *args): method = getattr(object, methodname) realresult = method(*args) @@ -505,6 +511,28 @@ class UnicodeTest(string_tests.CommonTest, text = 'abc def' self.assertIs(text.replace(pattern, pattern), text) + def test_repeat_id_preserving(self): + a = '123abc1@' + b = '456zyx-+' + self.assertEqual(id(a), id(a)) + self.assertNotEqual(id(a), id(b)) + self.assertNotEqual(id(a), id(a * -4)) + self.assertNotEqual(id(a), id(a * 0)) + self.assertEqual(id(a), id(a * 1)) + self.assertEqual(id(a), id(1 * a)) + self.assertNotEqual(id(a), id(a * 2)) + + class SubStr(str): + pass + + s = SubStr('qwerty()') + self.assertEqual(id(s), id(s)) + self.assertNotEqual(id(s), id(s * -4)) + self.assertNotEqual(id(s), id(s * 0)) + self.assertNotEqual(id(s), id(s * 1)) + self.assertNotEqual(id(s), id(1 * s)) + self.assertNotEqual(id(s), id(s * 2)) + def test_bytes_comparison(self): with warnings_helper.check_warnings(): warnings.simplefilter('ignore', BytesWarning) @@ -728,6 +756,16 @@ class UnicodeTest(string_tests.CommonTest, self.assertFalse("©".isidentifier()) self.assertFalse("0".isidentifier()) + @support.cpython_only + @support.requires_legacy_unicode_capi + def test_isidentifier_legacy(self): + import _testcapi + u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' + self.assertTrue(u.isidentifier()) + with warnings_helper.check_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_isprintable(self): @@ -1103,6 +1141,12 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('{0:^8s}'.format('result'), ' result ') self.assertEqual('{0:^9s}'.format('result'), ' result ') self.assertEqual('{0:^10s}'.format('result'), ' result ') + self.assertEqual('{0:8s}'.format('result'), 'result ') + self.assertEqual('{0:0s}'.format('result'), 'result') + self.assertEqual('{0:08s}'.format('result'), 'result00') + self.assertEqual('{0:<08s}'.format('result'), 'result00') + self.assertEqual('{0:>08s}'.format('result'), '00result') + self.assertEqual('{0:^08s}'.format('result'), '0result0') self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999) self.assertEqual('{0:10000}'.format(''), ' ' * 10000) self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000) @@ -1230,8 +1274,11 @@ class UnicodeTest(string_tests.CommonTest, 0, 1, 2, 3, 4, 5, 6, 7) # string format spec errors - self.assertRaises(ValueError, "{0:-s}".format, '') - self.assertRaises(ValueError, format, "", "-") + sign_msg = "Sign not allowed in string format specifier" + self.assertRaisesRegex(ValueError, sign_msg, "{0:-s}".format, '') + self.assertRaisesRegex(ValueError, sign_msg, format, "", "-") + space_msg = "Space not allowed in string format specifier" + self.assertRaisesRegex(ValueError, space_msg, "{: }".format, '') self.assertRaises(ValueError, "{0:=s}".format, '') # Alternate formatting is not supported @@ -1789,7 +1836,7 @@ class UnicodeTest(string_tests.CommonTest, # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, # only the start byte and the continuation byte(s) are now considered # invalid, instead of the number of bytes specified by the start byte. - # See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95, + # See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95, # table 3-8, Row 2) for more information about the algorithm used. FFFD = '\ufffd' sequences = [ @@ -2247,22 +2294,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(("abc" "def" "ghi"), "abcdefghi") self.assertEqual(("abc" "def" "ghi"), "abcdefghi") - def test_printing(self): - class BitBucket: - def write(self, text): - pass - - out = BitBucket() - print('abc', file=out) - print('abc', 'def', file=out) - print('abc', 'def', file=out) - print('abc', 'def', file=out) - print('abc\n', file=out) - print('abc\n', end=' ', file=out) - print('abc\n', end=' ', file=out) - print('def\n', file=out) - print('def\n', file=out) - def test_ucs4(self): x = '\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") @@ -2400,6 +2431,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(len(args), 1) @support.cpython_only + @support.requires_legacy_unicode_capi def test_resize(self): from _testcapi import getargs_u for length in range(1, 100, 7): @@ -2407,12 +2439,14 @@ class UnicodeTest(string_tests.CommonTest, text = 'a' * length + 'b' # fill wstr internal field - abc = getargs_u(text) + with self.assertWarns(DeprecationWarning): + abc = getargs_u(text) self.assertEqual(abc, text) # resize text: wstr field must be cleared and then recomputed text += 'c' - abcdef = getargs_u(text) + with self.assertWarns(DeprecationWarning): + abcdef = getargs_u(text) self.assertNotEqual(abc, abcdef) self.assertEqual(abcdef, text) @@ -2496,6 +2530,66 @@ class UnicodeTest(string_tests.CommonTest, support.check_free_after_iterating(self, iter, str) support.check_free_after_iterating(self, reversed, str) + def test_check_encoding_errors(self): + # bpo-37388: str(bytes) and str.decode() must check encoding and errors + # arguments in dev mode + encodings = ('ascii', 'utf8', 'latin1') + invalid = 'Boom, Shaka Laka, Boom!' + code = textwrap.dedent(f''' + import sys + encodings = {encodings!r} + + for data in (b'', b'short string'): + try: + str(data, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(21) + + try: + str(data, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + for encoding in encodings: + try: + str(data, encoding, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + for data in ('', 'short string'): + try: + data.encode(encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(23) + + try: + data.encode(errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + for encoding in encodings: + try: + data.encode(encoding, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + sys.exit(10) + ''') + proc = assert_python_failure('-X', 'dev', '-c', code) + self.assertEqual(proc.rc, 10, proc) + class CAPITest(unittest.TestCase): @@ -2503,11 +2597,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): import_helper.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): @@ -2807,15 +2903,43 @@ class CAPITest(unittest.TestCase): for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: l = len(s) - self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') - self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + self.assertEqual(unicode_asucs4(s, l, True), s+'\0') + self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, True) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, False) s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') + + # Test PyUnicode_AsUTF8() + @support.cpython_only + def test_asutf8(self): + from _testcapi import unicode_asutf8 + + bmp = '\u0100' + bmp2 = '\uffff' + nonbmp = chr(0x10ffff) + + self.assertEqual(unicode_asutf8(bmp), b'\xc4\x80') + self.assertEqual(unicode_asutf8(bmp2), b'\xef\xbf\xbf') + self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') + self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') + + # Test PyUnicode_AsUTF8AndSize() + @support.cpython_only + def test_asutf8andsize(self): + from _testcapi import unicode_asutf8andsize + + bmp = '\u0100' + bmp2 = '\uffff' + nonbmp = chr(0x10ffff) + + self.assertEqual(unicode_asutf8andsize(bmp), (b'\xc4\x80', 2)) + self.assertEqual(unicode_asutf8andsize(bmp2), (b'\xef\xbf\xbf', 3)) + self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') # Test PyUnicode_FindChar() @support.cpython_only @@ -2884,32 +3008,38 @@ class CAPITest(unittest.TestCase): self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) @support.cpython_only + @support.requires_legacy_unicode_capi def test_encode_decimal(self): from _testcapi import unicode_encodedecimal - self.assertEqual(unicode_encodedecimal('123'), - b'123') - self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), - b'3.14') - self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"), - b' 3.14 ') - self.assertRaises(UnicodeEncodeError, - unicode_encodedecimal, "123\u20ac", "strict") - self.assertRaisesRegex( - ValueError, - "^'decimal' codec can't encode character", - unicode_encodedecimal, "123\u20ac", "replace") + with warnings_helper.check_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual(unicode_encodedecimal('123'), + b'123') + self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), + b'3.14') + self.assertEqual(unicode_encodedecimal( + "\N{EM SPACE}3.14\N{EN SPACE}"), b' 3.14 ') + self.assertRaises(UnicodeEncodeError, + unicode_encodedecimal, "123\u20ac", "strict") + self.assertRaisesRegex( + ValueError, + "^'decimal' codec can't encode character", + unicode_encodedecimal, "123\u20ac", "replace") @support.cpython_only + @support.requires_legacy_unicode_capi def test_transform_decimal(self): from _testcapi import unicode_transformdecimaltoascii as transform_decimal - self.assertEqual(transform_decimal('123'), - '123') - self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), - '3.14') - self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), - "\N{EM SPACE}3.14\N{EN SPACE}") - self.assertEqual(transform_decimal('123\u20ac'), - '123\u20ac') + with warnings_helper.check_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + "\N{EM SPACE}3.14\N{EN SPACE}") + self.assertEqual(transform_decimal('123\u20ac'), + '123\u20ac') @support.cpython_only def test_pep393_utf8_caching_bug(self): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index c4bb3e186..169aaaf14 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,9 +5,9 @@ import sys import unittest import warnings from unicodedata import normalize -from test import support from test.support import os_helper + filenames = [ '1_abc', '2_ascii', @@ -151,7 +151,8 @@ class UnicodeFileTests(unittest.TestCase): sf0 = set(self.files) with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - f1 = os.listdir(os_helper.TESTFN.encode(sys.getfilesystemencoding())) + f1 = os.listdir(os_helper.TESTFN.encode( + sys.getfilesystemencoding())) f2 = os.listdir(os_helper.TESTFN) sf2 = set(os.path.join(os_helper.TESTFN, f) for f in f2) self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2)) @@ -163,7 +164,8 @@ class UnicodeFileTests(unittest.TestCase): os.rename("tmp", name) def test_directory(self): - dirname = os.path.join(os_helper.TESTFN, 'Gr\xfc\xdf-\u66e8\u66e9\u66eb') + dirname = os.path.join(os_helper.TESTFN, + 'Gr\xfc\xdf-\u66e8\u66e9\u66eb') filename = '\xdf-\u66e8\u66e9\u66eb' with os_helper.temp_cwd(dirname): with open(filename, 'wb') as f: @@ -188,15 +190,5 @@ class UnicodeNFKDFileTests(UnicodeFileTests): normal_form = 'NFKD' -def test_main(): - support.run_unittest( - UnicodeFileTests, - UnicodeNFCFileTests, - UnicodeNFDFileTests, - UnicodeNFKCFileTests, - UnicodeNFKDFileTests, - ) - - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 7bd5e5da5..9e611caf6 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -26,9 +26,11 @@ class PEP3131Test(unittest.TestCase): def test_invalid(self): try: from test import badsyntax_3131 - except SyntaxError as s: - self.assertEqual(str(s), - "invalid character in identifier (badsyntax_3131.py, line 2)") + except SyntaxError as err: + self.assertEqual(str(err), + "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") + self.assertEqual(err.lineno, 2) + self.assertEqual(err.offset, 1) else: self.fail("expected exception didn't occur")