diff --git a/Lib/gettext.py b/Lib/gettext.py index 62cff81b7..6c11ab2b1 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -41,14 +41,10 @@ internationalized, to the local language and cultural habits. # to do binary searches and lazy initializations. Or you might want to use # the undocumented double-hash algorithm for .mo files with hash tables, but # you'll need to study the GNU gettext code to do this. -# -# - Support Solaris .mo file formats. Unfortunately, we've been unable to -# find this format documented anywhere. import operator import os -import re import sys @@ -70,22 +66,26 @@ _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale') # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y -_token_pattern = re.compile(r""" - (?P[ \t]+) | # spaces and horizontal tabs - (?P[0-9]+\b) | # decimal integer - (?Pn\b) | # only n is allowed - (?P[()]) | - (?P[-*/%+?:]|[>, - # <=, >=, ==, !=, &&, ||, - # ? : - # unary and bitwise ops - # not allowed - (?P\w+|.) # invalid token - """, re.VERBOSE|re.DOTALL) - +_token_pattern = None def _tokenize(plural): - for mo in re.finditer(_token_pattern, plural): + global _token_pattern + if _token_pattern is None: + import re + _token_pattern = re.compile(r""" + (?P[ \t]+) | # spaces and horizontal tabs + (?P[0-9]+\b) | # decimal integer + (?Pn\b) | # only n is allowed + (?P[()]) | + (?P[-*/%+?:]|[>, + # <=, >=, ==, !=, &&, ||, + # ? : + # unary and bitwise ops + # not allowed + (?P\w+|.) # invalid token + """, re.VERBOSE|re.DOTALL) + + for mo in _token_pattern.finditer(plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue @@ -648,7 +648,7 @@ def npgettext(context, msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fc..38cf16d21 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __all__ = ['Option', __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -74,7 +74,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import sys, os -import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +87,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): @@ -263,6 +251,7 @@ class HelpFormatter: Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ + import textwrap text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, @@ -319,6 +308,7 @@ class HelpFormatter: indent_first = 0 result.append(opts) if option.help: + import textwrap help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 0653bb762..f40690829 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -6,13 +6,12 @@ import unittest.mock from functools import partial from test import support -from test.support import os_helper +from test.support import cpython_only, os_helper +from test.support.import_helper import ensure_lazy_imports # TODO: # - Add new tests, for example for "dgettext" -# - Remove dummy tests, for example testing for single and double quotes -# has no sense, it would have if we were testing a parser (i.e. pygettext) # - Tests should have only one assert. GNU_MO_DATA = b'''\ @@ -231,30 +230,6 @@ class GettextTestCase1(GettextBaseTest): eq(pgettext('my other context', 'nudge nudge'), 'wink wink (in "my other context")') - def test_double_quotes(self): - eq = self.assertEqual - # double quotes - eq(_("albatross"), 'albatross') - eq(_("mullusk"), 'bacon') - eq(_(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove') - eq(_(r"nudge nudge"), 'wink wink') - - def test_triple_single_quotes(self): - eq = self.assertEqual - # triple single quotes - eq(_('''albatross'''), 'albatross') - eq(_('''mullusk'''), 'bacon') - eq(_(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove') - eq(_(r'''nudge nudge'''), 'wink wink') - - def test_triple_double_quotes(self): - eq = self.assertEqual - # triple double quotes - eq(_("""albatross"""), 'albatross') - eq(_("""mullusk"""), 'bacon') - eq(_(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove') - eq(_(r"""nudge nudge"""), 'wink wink') - def test_multiline_strings(self): eq = self.assertEqual # multiline strings @@ -367,30 +342,6 @@ class GettextTestCase2(GettextBaseTest): eq(gettext.dpgettext('gettext', 'my other context', 'nudge nudge'), 'wink wink (in "my other context")') - def test_double_quotes(self): - eq = self.assertEqual - # double quotes - eq(self._("albatross"), 'albatross') - eq(self._("mullusk"), 'bacon') - eq(self._(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove') - eq(self._(r"nudge nudge"), 'wink wink') - - def test_triple_single_quotes(self): - eq = self.assertEqual - # triple single quotes - eq(self._('''albatross'''), 'albatross') - eq(self._('''mullusk'''), 'bacon') - eq(self._(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove') - eq(self._(r'''nudge nudge'''), 'wink wink') - - def test_triple_double_quotes(self): - eq = self.assertEqual - # triple double quotes - eq(self._("""albatross"""), 'albatross') - eq(self._("""mullusk"""), 'bacon') - eq(self._(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove') - eq(self._(r"""nudge nudge"""), 'wink wink') - def test_multiline_strings(self): eq = self.assertEqual # multiline strings @@ -434,8 +385,7 @@ class PluralFormsTests: x = ngettext(singular, plural, None) self.assertEqual(x, tplural) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms(self): self._test_plural_forms( self.ngettext, self.gettext, @@ -446,8 +396,7 @@ class PluralFormsTests: '%d file deleted', '%d files deleted', '%d file deleted', '%d files deleted') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms(self): ngettext = partial(self.npgettext, 'With context') gettext = partial(self.pgettext, 'With context') @@ -460,8 +409,7 @@ class PluralFormsTests: '%d file deleted', '%d files deleted', '%d file deleted', '%d files deleted') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_wrong_context_forms(self): self._test_plural_forms( partial(self.npgettext, 'Unknown context'), @@ -494,8 +442,7 @@ class GNUTranslationsWithDomainPluralFormsTestCase(PluralFormsTests, GettextBase self.pgettext = partial(gettext.dpgettext, 'gettext') self.npgettext = partial(gettext.dnpgettext, 'gettext') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms_wrong_domain(self): self._test_plural_forms( partial(gettext.dngettext, 'unknown'), @@ -504,8 +451,7 @@ class GNUTranslationsWithDomainPluralFormsTestCase(PluralFormsTests, GettextBase 'There is %s file', 'There are %s files', numbers_only=False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms_wrong_domain(self): self._test_plural_forms( partial(gettext.dnpgettext, 'unknown', 'With context'), @@ -526,8 +472,7 @@ class GNUTranslationsClassPluralFormsTestCase(PluralFormsTests, GettextBaseTest) self.pgettext = t.pgettext self.npgettext = t.npgettext - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms_null_translations(self): t = gettext.NullTranslations() self._test_plural_forms( @@ -536,8 +481,7 @@ class GNUTranslationsClassPluralFormsTestCase(PluralFormsTests, GettextBaseTest) 'There is %s file', 'There are %s files', numbers_only=False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms_null_translations(self): t = gettext.NullTranslations() self._test_plural_forms( @@ -630,6 +574,7 @@ class PluralFormsInternalTestCase(unittest.TestCase): s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "01233333333444444444444444444444444444444444444444444444444444444444444444444444444444444444444444445553333333344444444444444444444444444444444444444444444444444444444444444444444444444444444444444444") + @support.skip_wasi_stack_overflow() def test_security(self): raises = self.assertRaises # Test for a dangerous expression @@ -994,6 +939,17 @@ class MiscTestCase(unittest.TestCase): support.check__all__(self, gettext, not_exported={'c2py', 'ENOENT'}) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) + + +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index c68214961..e476e4727 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -14,8 +14,9 @@ import unittest from io import StringIO from test import support -from test.support import os_helper +from test.support import cpython_only, os_helper from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.import_helper import ensure_lazy_imports import optparse from optparse import make_option, Option, \ @@ -1655,6 +1656,10 @@ class MiscTestCase(unittest.TestCase): not_exported = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} support.check__all__(self, optparse, not_exported=not_exported) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("optparse", {"textwrap"}) + class TestTranslations(TestTranslationsBase): def test_translations(self): diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index dfbc2b93d..aca1f4276 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ How *do* you spell that odd word, anyways? # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ How *do* you spell that odd word, anyways? def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], @@ -765,10 +765,67 @@ some (including a hanging indent).''' # of IndentTestCase! class DedentTestCase(unittest.TestCase): + def test_type_error(self): + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(0) + + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(b'') + def assertUnchanged(self, text): """assert that dedent() has no effect on 'text'""" self.assertEqual(text, dedent(text)) + def test_dedent_whitespace(self): + # The empty string. + text = "" + self.assertUnchanged(text) + + # Only spaces. + text = " " + expect = "" + self.assertEqual(expect, dedent(text)) + + # Only tabs. + text = "\t\t\t\t" + expect = "" + self.assertEqual(expect, dedent(text)) + + # A mixture. + text = " \t \t\t \t " + expect = "" + self.assertEqual(expect, dedent(text)) + + # ASCII whitespace. + text = "\f\n\r\t\v " + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # One newline. + text = "\n" + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # Windows-style newlines. + text = "\r\n" * 5 + expect = "\n" * 5 + self.assertEqual(expect, dedent(text)) + + # Whitespace mixture. + text = " \n\t\n \n\t\t\n\n\n " + expect = "\n\n\n\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Lines consisting only of whitespace are always normalised + text = "a\n \n\t\n" + expect = "a\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Whitespace characters on non-empty lines are retained + text = "a\r\n\r\n\r\n" + expect = "a\r\n\n\n" + self.assertEqual(expect, dedent(text)) + def test_dedent_nomargin(self): # No lines indented. text = "Hello there.\nHow are you?\nOh good, I'm glad." diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 7ca393d1c..41366fbf4 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -2,7 +2,7 @@ """ # Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. +# Copyright (C) 2002 Python Software Foundation. # Written by Greg Ward import re @@ -86,7 +86,7 @@ class TextWrapper: -(?: (?<=%(lt)s{2}-) | (?<=%(lt)s-%(lt)s-)) (?= %(lt)s -? %(lt)s) | # end of word - (?=%(ws)s|\Z) + (?=%(ws)s|\z) | # em-dash (?<=%(wp)s) (?=-{2,}\w) ) @@ -107,7 +107,7 @@ class TextWrapper: sentence_end_re = re.compile(r'[a-z]' # lowercase letter r'[\.\!\?]' # sentence-ending punct. r'[\"\']?' # optional end-of-quote - r'\Z') # end of chunk + r'\z') # end of chunk def __init__(self, width=70, @@ -211,7 +211,7 @@ class TextWrapper: # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: @@ -413,9 +413,6 @@ def shorten(text, width, **kwargs): # -- Loosely related functionality ------------------------------------- -_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) -_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) - def dedent(text): """Remove any common leading whitespace from every line in `text`. @@ -429,42 +426,22 @@ def dedent(text): Entirely blank lines are normalized to a newline character. """ - # Look for the longest leading string of spaces and tabs common to - # all lines. - margin = None - text = _whitespace_only_re.sub('', text) - indents = _leading_whitespace_re.findall(text) - for indent in indents: - if margin is None: - margin = indent + try: + lines = text.split('\n') + except (AttributeError, TypeError): + msg = f'expected str object, not {type(text).__qualname__!r}' + raise TypeError(msg) from None - # Current line more deeply indented than previous winner: - # no change (previous winner is still on top). - elif indent.startswith(margin): - pass + # Get length of leading whitespace, inspired by ``os.path.commonprefix()``. + non_blank_lines = [l for l in lines if l and not l.isspace()] + l1 = min(non_blank_lines, default='') + l2 = max(non_blank_lines, default='') + margin = 0 + for margin, c in enumerate(l1): + if c != l2[margin] or c not in ' \t': + break - # Current line consistent with and no deeper than previous winner: - # it's the new winner. - elif margin.startswith(indent): - margin = indent - - # Find the largest common whitespace between current line and previous - # winner. - else: - for i, (x, y) in enumerate(zip(margin, indent)): - if x != y: - margin = margin[:i] - break - - # sanity check (testing/debugging only) - if 0 and margin: - for line in text.split("\n"): - assert not line or line.startswith(margin), \ - "line = %r, margin = %r" % (line, margin) - - if margin: - text = re.sub(r'(?m)^' + margin, '', text) - return text + return '\n'.join([l[margin:] if not l.isspace() else '' for l in lines]) def indent(text, prefix, predicate=None): @@ -475,19 +452,20 @@ def indent(text, prefix, predicate=None): it will default to adding 'prefix' to all non-empty lines that do not consist solely of whitespace characters. """ - if predicate is None: - # str.splitlines(True) doesn't produce empty string. - # ''.splitlines(True) => [] - # 'foo\n'.splitlines(True) => ['foo\n'] - # So we can use just `not s.isspace()` here. - predicate = lambda s: not s.isspace() - prefixed_lines = [] - for line in text.splitlines(True): - if predicate(line): - prefixed_lines.append(prefix) - prefixed_lines.append(line) - + if predicate is None: + # str.splitlines(keepends=True) doesn't produce the empty string, + # so we need to use `str.isspace()` rather than a truth test. + # Inlining the predicate leads to a ~30% performance improvement. + for line in text.splitlines(True): + if not line.isspace(): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + else: + for line in text.splitlines(True): + if predicate(line): + prefixed_lines.append(prefix) + prefixed_lines.append(line) return ''.join(prefixed_lines)