Update optparse,gettext,textwrap from 3.14.2 (#6896)

* Update optparse from 3.14.2

* Update `gettext`

* Update `textwrap`

Lib/gettext.py

@@ -41,14 +41,10 @@ internationalized, to the local language and cultural habits.
 #   to do binary searches and lazy initializations.  Or you might want to use
 #   the undocumented double-hash algorithm for .mo files with hash tables, but
 #   you'll need to study the GNU gettext code to do this.
-#
-# - Support Solaris .mo file formats.  Unfortunately, we've been unable to
-#   find this format documented anywhere.
 
 
 import operator
 import os
-import re
 import sys
 
 
@@ -70,22 +66,26 @@ _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
 # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms
 # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y
 
-_token_pattern = re.compile(r"""
-        (?P<WHITESPACES>[ \t]+)                    | # spaces and horizontal tabs
-        (?P<NUMBER>[0-9]+\b)                       | # decimal integer
-        (?P<NAME>n\b)                              | # only n is allowed
-        (?P<PARENTHESIS>[()])                      |
-        (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,
-                                                     # <=, >=, ==, !=, &&, ||,
-                                                     # ? :
-                                                     # unary and bitwise ops
-                                                     # not allowed
-        (?P<INVALID>\w+|.)                           # invalid token
-    """, re.VERBOSE|re.DOTALL)
-
+_token_pattern = None
 
 def _tokenize(plural):
-    for mo in re.finditer(_token_pattern, plural):
+    global _token_pattern
+    if _token_pattern is None:
+        import re
+        _token_pattern = re.compile(r"""
+                (?P<WHITESPACES>[ \t]+)                    | # spaces and horizontal tabs
+                (?P<NUMBER>[0-9]+\b)                       | # decimal integer
+                (?P<NAME>n\b)                              | # only n is allowed
+                (?P<PARENTHESIS>[()])                      |
+                (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,
+                                                             # <=, >=, ==, !=, &&, ||,
+                                                             # ? :
+                                                             # unary and bitwise ops
+                                                             # not allowed
+                (?P<INVALID>\w+|.)                           # invalid token
+            """, re.VERBOSE|re.DOTALL)
+
+    for mo in _token_pattern.finditer(plural):
         kind = mo.lastgroup
         if kind == 'WHITESPACES':
             continue
@@ -648,7 +648,7 @@ def npgettext(context, msgid1, msgid2, n):
 #    import gettext
 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 #    _ = cat.gettext
-#    print _('Hello World')
+#    print(_('Hello World'))
 
 # The resulting catalog object currently don't support access through a
 # dictionary API, which was supported (but apparently unused) in GNOME

Lib/optparse.py

@@ -43,7 +43,7 @@ __all__ = ['Option',
 
 __copyright__ = """
 Copyright (c) 2001-2006 Gregory P. Ward.  All rights reserved.
-Copyright (c) 2002-2006 Python Software Foundation.  All rights reserved.
+Copyright (c) 2002 Python Software Foundation.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -74,7 +74,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
 
 import sys, os
-import textwrap
+from gettext import gettext as _, ngettext
+
 
 def _repr(self):
     return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self)
@@ -86,19 +87,6 @@ def _repr(self):
 #   Id: help.py 527 2006-07-23 15:21:30Z greg
 #   Id: errors.py 509 2006-04-20 00:58:24Z gward
 
-try:
-    from gettext import gettext, ngettext
-except ImportError:
-    def gettext(message):
-        return message
-
-    def ngettext(singular, plural, n):
-        if n == 1:
-            return singular
-        return plural
-
-_ = gettext
-
 
 class OptParseError (Exception):
     def __init__(self, msg):
@@ -263,6 +251,7 @@ class HelpFormatter:
         Format a paragraph of free-form text for inclusion in the
         help output at the current indentation level.
         """
+        import textwrap
         text_width = max(self.width - self.current_indent, 11)
         indent = " "*self.current_indent
         return textwrap.fill(text,
@@ -319,6 +308,7 @@ class HelpFormatter:
             indent_first = 0
         result.append(opts)
         if option.help:
+            import textwrap
             help_text = self.expand_default(option)
             help_lines = textwrap.wrap(help_text, self.help_width)
             result.append("%*s%s\n" % (indent_first, "", help_lines[0]))

Lib/test/test_gettext.py

@@ -6,13 +6,12 @@ import unittest.mock
 from functools import partial
 
 from test import support
-from test.support import os_helper
+from test.support import cpython_only, os_helper
+from test.support.import_helper import ensure_lazy_imports
 
 
 # TODO:
 #  - Add new tests, for example for "dgettext"
-#  - Remove dummy tests, for example testing for single and double quotes
-#    has no sense, it would have if we were testing a parser (i.e. pygettext)
 #  - Tests should have only one assert.
 
 GNU_MO_DATA = b'''\
@@ -231,30 +230,6 @@ class GettextTestCase1(GettextBaseTest):
         eq(pgettext('my other context', 'nudge nudge'),
            'wink wink (in "my other context")')
 
-    def test_double_quotes(self):
-        eq = self.assertEqual
-        # double quotes
-        eq(_("albatross"), 'albatross')
-        eq(_("mullusk"), 'bacon')
-        eq(_(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove')
-        eq(_(r"nudge nudge"), 'wink wink')
-
-    def test_triple_single_quotes(self):
-        eq = self.assertEqual
-        # triple single quotes
-        eq(_('''albatross'''), 'albatross')
-        eq(_('''mullusk'''), 'bacon')
-        eq(_(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove')
-        eq(_(r'''nudge nudge'''), 'wink wink')
-
-    def test_triple_double_quotes(self):
-        eq = self.assertEqual
-        # triple double quotes
-        eq(_("""albatross"""), 'albatross')
-        eq(_("""mullusk"""), 'bacon')
-        eq(_(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove')
-        eq(_(r"""nudge nudge"""), 'wink wink')
-
     def test_multiline_strings(self):
         eq = self.assertEqual
         # multiline strings
@@ -367,30 +342,6 @@ class GettextTestCase2(GettextBaseTest):
         eq(gettext.dpgettext('gettext', 'my other context', 'nudge nudge'),
            'wink wink (in "my other context")')
 
-    def test_double_quotes(self):
-        eq = self.assertEqual
-        # double quotes
-        eq(self._("albatross"), 'albatross')
-        eq(self._("mullusk"), 'bacon')
-        eq(self._(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove')
-        eq(self._(r"nudge nudge"), 'wink wink')
-
-    def test_triple_single_quotes(self):
-        eq = self.assertEqual
-        # triple single quotes
-        eq(self._('''albatross'''), 'albatross')
-        eq(self._('''mullusk'''), 'bacon')
-        eq(self._(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove')
-        eq(self._(r'''nudge nudge'''), 'wink wink')
-
-    def test_triple_double_quotes(self):
-        eq = self.assertEqual
-        # triple double quotes
-        eq(self._("""albatross"""), 'albatross')
-        eq(self._("""mullusk"""), 'bacon')
-        eq(self._(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove')
-        eq(self._(r"""nudge nudge"""), 'wink wink')
-
     def test_multiline_strings(self):
         eq = self.assertEqual
         # multiline strings
@@ -434,8 +385,7 @@ class PluralFormsTests:
                 x = ngettext(singular, plural, None)
             self.assertEqual(x, tplural)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_forms(self):
         self._test_plural_forms(
             self.ngettext, self.gettext,
@@ -446,8 +396,7 @@ class PluralFormsTests:
             '%d file deleted', '%d files deleted',
             '%d file deleted', '%d files deleted')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_context_forms(self):
         ngettext = partial(self.npgettext, 'With context')
         gettext = partial(self.pgettext, 'With context')
@@ -460,8 +409,7 @@ class PluralFormsTests:
             '%d file deleted', '%d files deleted',
             '%d file deleted', '%d files deleted')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_wrong_context_forms(self):
         self._test_plural_forms(
             partial(self.npgettext, 'Unknown context'),
@@ -494,8 +442,7 @@ class GNUTranslationsWithDomainPluralFormsTestCase(PluralFormsTests, GettextBase
         self.pgettext = partial(gettext.dpgettext, 'gettext')
         self.npgettext = partial(gettext.dnpgettext, 'gettext')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_forms_wrong_domain(self):
         self._test_plural_forms(
             partial(gettext.dngettext, 'unknown'),
@@ -504,8 +451,7 @@ class GNUTranslationsWithDomainPluralFormsTestCase(PluralFormsTests, GettextBase
             'There is %s file', 'There are %s files',
             numbers_only=False)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_context_forms_wrong_domain(self):
         self._test_plural_forms(
             partial(gettext.dnpgettext, 'unknown', 'With context'),
@@ -526,8 +472,7 @@ class GNUTranslationsClassPluralFormsTestCase(PluralFormsTests, GettextBaseTest)
         self.pgettext = t.pgettext
         self.npgettext = t.npgettext
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_forms_null_translations(self):
         t = gettext.NullTranslations()
         self._test_plural_forms(
@@ -536,8 +481,7 @@ class GNUTranslationsClassPluralFormsTestCase(PluralFormsTests, GettextBaseTest)
             'There is %s file', 'There are %s files',
             numbers_only=False)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_plural_context_forms_null_translations(self):
         t = gettext.NullTranslations()
         self._test_plural_forms(
@@ -630,6 +574,7 @@ class PluralFormsInternalTestCase(unittest.TestCase):
         s = ''.join([ str(f(x)) for x in range(200) ])
         eq(s, "01233333333444444444444444444444444444444444444444444444444444444444444444444444444444444444444444445553333333344444444444444444444444444444444444444444444444444444444444444444444444444444444444444444")
 
+    @support.skip_wasi_stack_overflow()
     def test_security(self):
         raises = self.assertRaises
         # Test for a dangerous expression
@@ -994,6 +939,17 @@ class MiscTestCase(unittest.TestCase):
         support.check__all__(self, gettext,
                              not_exported={'c2py', 'ENOENT'})
 
+    @cpython_only
+    def test_lazy_import(self):
+        ensure_lazy_imports("gettext", {"re", "warnings", "locale"})
+
+
+class TranslationFallbackTestCase(unittest.TestCase):
+    def test_translation_fallback(self):
+        with os_helper.temp_cwd() as tempdir:
+            t = gettext.translation('gettext', localedir=tempdir, fallback=True)
+            self.assertIsInstance(t, gettext.NullTranslations)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_optparse.py

@@ -14,8 +14,9 @@ import unittest
 
 from io import StringIO
 from test import support
-from test.support import os_helper
+from test.support import cpython_only, os_helper
 from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots
+from test.support.import_helper import ensure_lazy_imports
 
 import optparse
 from optparse import make_option, Option, \
@@ -1655,6 +1656,10 @@ class MiscTestCase(unittest.TestCase):
         not_exported = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'}
         support.check__all__(self, optparse, not_exported=not_exported)
 
+    @cpython_only
+    def test_lazy_import(self):
+        ensure_lazy_imports("optparse", {"textwrap"})
+
 
 class TestTranslations(TestTranslationsBase):
     def test_translations(self):

Lib/test/test_textwrap.py

@@ -605,7 +605,7 @@ How *do* you spell that odd word, anyways?
         # bug 1146.  Prevent a long word to be wrongly wrapped when the
         # preceding word is exactly one character shorter than the width
         self.check_wrap(self.text, 12,
-                        ['Did you say ',
+                        ['Did you say',
                          '"supercalifr',
                          'agilisticexp',
                          'ialidocious?',
@@ -633,7 +633,7 @@ How *do* you spell that odd word, anyways?
 
     def test_max_lines_long(self):
         self.check_wrap(self.text, 12,
-                        ['Did you say ',
+                        ['Did you say',
                          '"supercalifr',
                          'agilisticexp',
                          '[...]'],
@@ -765,10 +765,67 @@ some (including a hanging indent).'''
 # of IndentTestCase!
 class DedentTestCase(unittest.TestCase):
 
+    def test_type_error(self):
+        with self.assertRaisesRegex(TypeError, "expected str object, not"):
+            dedent(0)
+
+        with self.assertRaisesRegex(TypeError, "expected str object, not"):
+            dedent(b'')
+
     def assertUnchanged(self, text):
         """assert that dedent() has no effect on 'text'"""
         self.assertEqual(text, dedent(text))
 
+    def test_dedent_whitespace(self):
+        # The empty string.
+        text = ""
+        self.assertUnchanged(text)
+
+        # Only spaces.
+        text = "    "
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # Only tabs.
+        text = "\t\t\t\t"
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # A mixture.
+        text = " \t  \t\t  \t "
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # ASCII whitespace.
+        text = "\f\n\r\t\v "
+        expect = "\n"
+        self.assertEqual(expect, dedent(text))
+
+        # One newline.
+        text = "\n"
+        expect = "\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Windows-style newlines.
+        text = "\r\n" * 5
+        expect = "\n" * 5
+        self.assertEqual(expect, dedent(text))
+
+        # Whitespace mixture.
+        text = "    \n\t\n  \n\t\t\n\n\n       "
+        expect = "\n\n\n\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Lines consisting only of whitespace are always normalised
+        text = "a\n \n\t\n"
+        expect = "a\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Whitespace characters on non-empty lines are retained
+        text = "a\r\n\r\n\r\n"
+        expect = "a\r\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
     def test_dedent_nomargin(self):
         # No lines indented.
         text = "Hello there.\nHow are you?\nOh good, I'm glad."

Lib/textwrap.py

@@ -2,7 +2,7 @@
 """
 
 # Copyright (C) 1999-2001 Gregory P. Ward.
-# Copyright (C) 2002, 2003 Python Software Foundation.
+# Copyright (C) 2002 Python Software Foundation.
 # Written by Greg Ward <gward@python.net>
 
 import re
@@ -86,7 +86,7 @@ class TextWrapper:
               -(?: (?<=%(lt)s{2}-) | (?<=%(lt)s-%(lt)s-))
               (?= %(lt)s -? %(lt)s)
             | # end of word
-              (?=%(ws)s|\Z)
+              (?=%(ws)s|\z)
             | # em-dash
               (?<=%(wp)s) (?=-{2,}\w)
             )
@@ -107,7 +107,7 @@ class TextWrapper:
     sentence_end_re = re.compile(r'[a-z]'             # lowercase letter
                                  r'[\.\!\?]'          # sentence-ending punct.
                                  r'[\"\']?'           # optional end-of-quote
-                                 r'\Z')               # end of chunk
+                                 r'\z')               # end of chunk
 
     def __init__(self,
                  width=70,
@@ -211,7 +211,7 @@ class TextWrapper:
 
         # If we're allowed to break long words, then do so: put as much
         # of the next chunk onto the current line as will fit.
-        if self.break_long_words:
+        if self.break_long_words and space_left > 0:
             end = space_left
             chunk = reversed_chunks[-1]
             if self.break_on_hyphens and len(chunk) > space_left:
@@ -413,9 +413,6 @@ def shorten(text, width, **kwargs):
 
 # -- Loosely related functionality -------------------------------------
 
-_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE)
-_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE)
-
 def dedent(text):
     """Remove any common leading whitespace from every line in `text`.
 
@@ -429,42 +426,22 @@ def dedent(text):
 
     Entirely blank lines are normalized to a newline character.
     """
-    # Look for the longest leading string of spaces and tabs common to
-    # all lines.
-    margin = None
-    text = _whitespace_only_re.sub('', text)
-    indents = _leading_whitespace_re.findall(text)
-    for indent in indents:
-        if margin is None:
-            margin = indent
+    try:
+        lines = text.split('\n')
+    except (AttributeError, TypeError):
+        msg = f'expected str object, not {type(text).__qualname__!r}'
+        raise TypeError(msg) from None
 
-        # Current line more deeply indented than previous winner:
-        # no change (previous winner is still on top).
-        elif indent.startswith(margin):
-            pass
+    # Get length of leading whitespace, inspired by ``os.path.commonprefix()``.
+    non_blank_lines = [l for l in lines if l and not l.isspace()]
+    l1 = min(non_blank_lines, default='')
+    l2 = max(non_blank_lines, default='')
+    margin = 0
+    for margin, c in enumerate(l1):
+        if c != l2[margin] or c not in ' \t':
+            break
 
-        # Current line consistent with and no deeper than previous winner:
-        # it's the new winner.
-        elif margin.startswith(indent):
-            margin = indent
-
-        # Find the largest common whitespace between current line and previous
-        # winner.
-        else:
-            for i, (x, y) in enumerate(zip(margin, indent)):
-                if x != y:
-                    margin = margin[:i]
-                    break
-
-    # sanity check (testing/debugging only)
-    if 0 and margin:
-        for line in text.split("\n"):
-            assert not line or line.startswith(margin), \
-                   "line = %r, margin = %r" % (line, margin)
-
-    if margin:
-        text = re.sub(r'(?m)^' + margin, '', text)
-    return text
+    return '\n'.join([l[margin:] if not l.isspace() else '' for l in lines])
 
 
 def indent(text, prefix, predicate=None):
@@ -475,19 +452,20 @@ def indent(text, prefix, predicate=None):
     it will default to adding 'prefix' to all non-empty lines that do not
     consist solely of whitespace characters.
     """
-    if predicate is None:
-        # str.splitlines(True) doesn't produce empty string.
-        #  ''.splitlines(True) => []
-        #  'foo\n'.splitlines(True) => ['foo\n']
-        # So we can use just `not s.isspace()` here.
-        predicate = lambda s: not s.isspace()
-
     prefixed_lines = []
-    for line in text.splitlines(True):
-        if predicate(line):
-            prefixed_lines.append(prefix)
-        prefixed_lines.append(line)
-
+    if predicate is None:
+        # str.splitlines(keepends=True) doesn't produce the empty string,
+        # so we need to use `str.isspace()` rather than a truth test.
+        # Inlining the predicate leads to a ~30% performance improvement.
+        for line in text.splitlines(True):
+            if not line.isspace():
+                prefixed_lines.append(prefix)
+            prefixed_lines.append(line)
+    else:
+        for line in text.splitlines(True):
+            if predicate(line):
+                prefixed_lines.append(prefix)
+            prefixed_lines.append(line)
     return ''.join(prefixed_lines)