Update NOP tests to 3.14.5 (#7902)

* Update `test_xxlimited.py` & `test_xxtestfuzz.py` to 3.14.5

* Add more NOP tests

* Remove `test_capi`

* Remove `test_cext`

Lib/test/test_asdl_parser.py

@@ -0,0 +1,131 @@
+"""Tests for the asdl parser in Parser/asdl.py"""
+
+import importlib.machinery
+import importlib.util
+import os
+from os.path import dirname
+import sys
+import sysconfig
+import unittest
+
+
+# This test is only relevant for from-source builds of Python.
+if not sysconfig.is_python_build():
+    raise unittest.SkipTest('test irrelevant for an installed Python')
+
+src_base = dirname(dirname(dirname(__file__)))
+parser_dir = os.path.join(src_base, 'Parser')
+
+
+class TestAsdlParser(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Loads the asdl module dynamically, since it's not in a real importable
+        # package.
+        # Parses Python.asdl into an ast.Module and run the check on it.
+        # There's no need to do this for each test method, hence setUpClass.
+        sys.path.insert(0, parser_dir)
+        loader = importlib.machinery.SourceFileLoader(
+                'asdl', os.path.join(parser_dir, 'asdl.py'))
+        spec = importlib.util.spec_from_loader('asdl', loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+        cls.asdl = module
+        cls.mod = cls.asdl.parse(os.path.join(parser_dir, 'Python.asdl'))
+        cls.assertTrue(cls.asdl.check(cls.mod), 'Module validation failed')
+
+    @classmethod
+    def tearDownClass(cls):
+        del sys.path[0]
+
+    def setUp(self):
+        # alias stuff from the class, for convenience
+        self.asdl = TestAsdlParser.asdl
+        self.mod = TestAsdlParser.mod
+        self.types = self.mod.types
+
+    def test_module(self):
+        self.assertEqual(self.mod.name, 'Python')
+        self.assertIn('stmt', self.types)
+        self.assertIn('expr', self.types)
+        self.assertIn('mod', self.types)
+
+    def test_definitions(self):
+        defs = self.mod.dfns
+        self.assertIsInstance(defs[0], self.asdl.Type)
+        self.assertIsInstance(defs[0].value, self.asdl.Sum)
+
+        self.assertIsInstance(self.types['withitem'], self.asdl.Product)
+        self.assertIsInstance(self.types['alias'], self.asdl.Product)
+
+    def test_product(self):
+        alias = self.types['alias']
+        self.assertEqual(
+            str(alias),
+            'Product([Field(identifier, name), Field(identifier, asname, quantifiers=[OPTIONAL])], '
+            '[Field(int, lineno), Field(int, col_offset), '
+            'Field(int, end_lineno, quantifiers=[OPTIONAL]), Field(int, end_col_offset, quantifiers=[OPTIONAL])])')
+
+    def test_attributes(self):
+        stmt = self.types['stmt']
+        self.assertEqual(len(stmt.attributes), 4)
+        self.assertEqual(repr(stmt.attributes[0]), 'Field(int, lineno)')
+        self.assertEqual(repr(stmt.attributes[1]), 'Field(int, col_offset)')
+        self.assertEqual(repr(stmt.attributes[2]), 'Field(int, end_lineno, quantifiers=[OPTIONAL])')
+        self.assertEqual(repr(stmt.attributes[3]), 'Field(int, end_col_offset, quantifiers=[OPTIONAL])')
+
+    def test_constructor_fields(self):
+        ehandler = self.types['excepthandler']
+        self.assertEqual(len(ehandler.types), 1)
+        self.assertEqual(len(ehandler.attributes), 4)
+
+        cons = ehandler.types[0]
+        self.assertIsInstance(cons, self.asdl.Constructor)
+        self.assertEqual(len(cons.fields), 3)
+
+        f0 = cons.fields[0]
+        self.assertEqual(f0.type, 'expr')
+        self.assertEqual(f0.name, 'type')
+        self.assertTrue(f0.opt)
+
+        f1 = cons.fields[1]
+        self.assertEqual(f1.type, 'identifier')
+        self.assertEqual(f1.name, 'name')
+        self.assertTrue(f1.opt)
+
+        f2 = cons.fields[2]
+        self.assertEqual(f2.type, 'stmt')
+        self.assertEqual(f2.name, 'body')
+        self.assertFalse(f2.opt)
+        self.assertTrue(f2.seq)
+
+    def test_visitor(self):
+        class CustomVisitor(self.asdl.VisitorBase):
+            def __init__(self):
+                super().__init__()
+                self.names_with_seq = []
+
+            def visitModule(self, mod):
+                for dfn in mod.dfns:
+                    self.visit(dfn)
+
+            def visitType(self, type):
+                self.visit(type.value)
+
+            def visitSum(self, sum):
+                for t in sum.types:
+                    self.visit(t)
+
+            def visitConstructor(self, cons):
+                for f in cons.fields:
+                    if f.seq:
+                        self.names_with_seq.append(cons.name)
+
+        v = CustomVisitor()
+        v.visit(self.types['mod'])
+        self.assertEqual(v.names_with_seq,
+                         ['Module', 'Module', 'Interactive', 'FunctionType'])
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_peg_generator/__init__.py

@@ -0,0 +1,12 @@
+import os.path
+from test import support
+from test.support import load_package_tests
+
+
+# Creating a virtual environment and building C extensions is slow
+support.requires('cpu')
+
+
+# Load all tests in package
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_peg_generator/__main__.py

@@ -0,0 +1,4 @@
+import unittest
+from . import load_tests
+
+unittest.main()

Lib/test/test_peg_generator/test_c_parser.py

@@ -0,0 +1,523 @@
+import contextlib
+import subprocess
+import sysconfig
+import textwrap
+import unittest
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+from test import test_tools
+from test import support
+from test.support import os_helper, import_helper
+from test.support.script_helper import assert_python_ok
+
+if support.check_cflags_pgo():
+    raise unittest.SkipTest("peg_generator test disabled under PGO build")
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.testutil import (
+        parse_string,
+        generate_parser_c_extension,
+        generate_c_parser_source,
+    )
+
+
+TEST_TEMPLATE = """
+tmp_dir = {extension_path!r}
+
+import ast
+import traceback
+import sys
+import unittest
+
+from test import test_tools
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.ast_dump import ast_dump
+
+sys.path.insert(0, tmp_dir)
+import parse
+
+class Tests(unittest.TestCase):
+
+    def check_input_strings_for_grammar(
+        self,
+        valid_cases = (),
+        invalid_cases = (),
+    ):
+        if valid_cases:
+            for case in valid_cases:
+                parse.parse_string(case, mode=0)
+
+        if invalid_cases:
+            for case in invalid_cases:
+                with self.assertRaises(SyntaxError):
+                    parse.parse_string(case, mode=0)
+
+    def verify_ast_generation(self, stmt):
+        expected_ast = ast.parse(stmt)
+        actual_ast = parse.parse_string(stmt, mode=1)
+        self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
+
+    def test_parse(self):
+        {test_source}
+
+unittest.main()
+"""
+
+
+@support.requires_subprocess()
+class TestCParser(unittest.TestCase):
+
+    _has_run = False
+
+    @classmethod
+    def setUpClass(cls):
+        if cls._has_run:
+            # Since gh-104798 (Use setuptools in peg-generator and reenable
+            # tests), this test case has been producing ref leaks. Initial
+            # debugging points to bug(s) in setuptools and/or importlib.
+            # See gh-105063 for more info.
+            raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks")
+        cls._has_run = True
+
+        # When running under regtest, a separate tempdir is used
+        # as the current directory and watched for left-overs.
+        # Reusing that as the base for temporary directories
+        # ensures everything is cleaned up properly and
+        # cleans up afterwards if not (with warnings).
+        cls.tmp_base = os.getcwd()
+        if os.path.samefile(cls.tmp_base, os_helper.SAVEDCWD):
+            cls.tmp_base = None
+        # Create a directory for the reuseable static library part of
+        # the pegen extension build process.  This greatly reduces the
+        # runtime overhead of spawning compiler processes.
+        cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
+        cls.addClassCleanup(shutil.rmtree, cls.library_dir)
+
+        with contextlib.ExitStack() as stack:
+            python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv"))
+            sitepackages = subprocess.check_output(
+                [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
+                text=True,
+            ).strip()
+            stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
+            cls.addClassCleanup(stack.pop_all().close)
+
+    @support.requires_venv_with_pip()
+    def setUp(self):
+        self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
+        cmd = support.missing_compiler_executable()
+        if cmd is not None:
+            self.skipTest("The %r command is not found" % cmd)
+        self.old_cwd = os.getcwd()
+        self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base)
+        self.enterContext(os_helper.change_cwd(self.tmp_path))
+
+    def tearDown(self):
+        os.chdir(self.old_cwd)
+        shutil.rmtree(self.tmp_path)
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(self._backup_config_vars)
+
+    def build_extension(self, grammar_source):
+        grammar = parse_string(grammar_source, GrammarParser)
+        # Because setUp() already changes the current directory to the
+        # temporary path, use a relative path here to prevent excessive
+        # path lengths when compiling.
+        generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir)
+
+    def run_test(self, grammar_source, test_source):
+        self.build_extension(grammar_source)
+        test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ")
+        assert_python_ok(
+            "-c",
+            TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source),
+        )
+
+    def test_c_parser(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }
+        stmt[stmt_ty]: a=expr_stmt { a }
+        expr_stmt[stmt_ty]: a=expression NEWLINE { _PyAST_Expr(a, EXTRA) }
+        expression[expr_ty]: ( l=expression '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) }
+                            | l=expression '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) }
+                            | t=term { t }
+                            )
+        term[expr_ty]: ( l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) }
+                    | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) }
+                    | f=factor { f }
+                    )
+        factor[expr_ty]: ('(' e=expression ')' { e }
+                        | a=atom { a }
+                        )
+        atom[expr_ty]: ( n=NAME { n }
+                    | n=NUMBER { n }
+                    | s=STRING { s }
+                    )
+        """
+        test_source = """
+        expressions = [
+            "4+5",
+            "4-5",
+            "4*5",
+            "1+4*5",
+            "1+4/5",
+            "(1+1) + (1+1)",
+            "(1+1) - (1+1)",
+            "(1+1) * (1+1)",
+            "(1+1) / (1+1)",
+        ]
+
+        for expr in expressions:
+            the_ast = parse.parse_string(expr, mode=1)
+            expected_ast = ast.parse(expr)
+            self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_lookahead(self) -> None:
+        grammar_source = """
+        start: NAME &NAME expr NEWLINE? ENDMARKER
+        expr: NAME | NUMBER
+        """
+        test_source = """
+        valid_cases = ["foo bar"]
+        invalid_cases = ["foo 34"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_negative_lookahead(self) -> None:
+        grammar_source = """
+        start: NAME !NAME expr NEWLINE? ENDMARKER
+        expr: NAME | NUMBER
+        """
+        test_source = """
+        valid_cases = ["foo 34"]
+        invalid_cases = ["foo bar"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_cut(self) -> None:
+        grammar_source = """
+        start: X ~ Y Z | X Q S
+        X: 'x'
+        Y: 'y'
+        Z: 'z'
+        Q: 'q'
+        S: 's'
+        """
+        test_source = """
+        valid_cases = ["x y z"]
+        invalid_cases = ["x q s"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_gather(self) -> None:
+        grammar_source = """
+        start: ';'.pass_stmt+ NEWLINE
+        pass_stmt: 'pass'
+        """
+        test_source = """
+        valid_cases = ["pass", "pass; pass"]
+        invalid_cases = ["pass;", "pass; pass;"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_left_recursion(self) -> None:
+        grammar_source = """
+        start: expr NEWLINE
+        expr: ('-' term | expr '+' term | term)
+        term: NUMBER
+        """
+        test_source = """
+        valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_advanced_left_recursive(self) -> None:
+        grammar_source = """
+        start: NUMBER | sign start
+        sign: ['-']
+        """
+        test_source = """
+        valid_cases = ["23", "-34"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_mutually_left_recursive(self) -> None:
+        grammar_source = """
+        start: foo 'E'
+        foo: bar 'A' | 'B'
+        bar: foo 'C' | 'D'
+        """
+        test_source = """
+        valid_cases = ["B E", "D A C A E"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_nasty_mutually_left_recursive(self) -> None:
+        grammar_source = """
+        start: target '='
+        target: maybe '+' | NAME
+        maybe: maybe '-' | target
+        """
+        test_source = """
+        valid_cases = ["x ="]
+        invalid_cases = ["x - + ="]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_return_stmt_noexpr_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
+        statement[stmt_ty]: simple_stmt
+        simple_stmt[stmt_ty]: small_stmt
+        small_stmt[stmt_ty]: return_stmt
+        return_stmt[stmt_ty]: a='return' NEWLINE { _PyAST_Return(NULL, EXTRA) }
+        """
+        test_source = """
+        stmt = "return"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_gather_action_ast(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=';'.pass_stmt+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA)}
+        """
+        test_source = """
+        stmt = "pass; pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_pass_stmt_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
+        statement[stmt_ty]: simple_stmt
+        simple_stmt[stmt_ty]: small_stmt
+        small_stmt[stmt_ty]: pass_stmt
+        pass_stmt[stmt_ty]: a='pass' NEWLINE { _PyAST_Pass(EXTRA) }
+        """
+        test_source = """
+        stmt = "pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_if_stmt_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
+        statement[asdl_stmt_seq*]:  a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmt
+
+        simple_stmt[asdl_stmt_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE {
+                                            (asdl_stmt_seq*)_PyPegen_seq_insert_in_front(p, a, b) }
+        further_small_stmt[stmt_ty]: ';' a=small_stmt { a }
+
+        block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a }
+
+        compound_stmt: if_stmt
+
+        if_stmt: 'if' a=full_expression ':' b=block { _PyAST_If(a, b, NULL, EXTRA) }
+
+        small_stmt[stmt_ty]: pass_stmt
+
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
+
+        full_expression: NAME
+        """
+        test_source = """
+        stmt = "pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_same_name_different_types(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=import_from+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena)}
+        import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from {
+                                _PyAST_ImportFrom(c->v.Name.id, d, 0, EXTRA) }
+                            | a='from' '.' 'import' c=import_as_names_from {
+                                _PyAST_ImportFrom(NULL, c, 1, EXTRA) }
+                            )
+        simple_name[expr_ty]: NAME
+        import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a }
+        import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) }
+        """
+        test_source = """
+        for stmt in ("from a import b as c", "from . import a as b"):
+            expected_ast = ast.parse(stmt)
+            actual_ast = parse.parse_string(stmt, mode=1)
+            self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_with_stmt_with_paren(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
+        statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
+        compound_stmt[stmt_ty]: with_stmt
+        with_stmt[stmt_ty]: (
+            a='with' '(' b[asdl_withitem_seq*]=','.with_item+ ')' ':' c=block {
+                _PyAST_With(b, (asdl_stmt_seq*) _PyPegen_singleton_seq(p, c), NULL, EXTRA) }
+        )
+        with_item[withitem_ty]: (
+            e=NAME o=['as' t=NAME { t }] { _PyAST_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) }
+        )
+        block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a }
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
+        """
+        test_source = """
+        stmt = "with (\\n    a as b,\\n    c as d\\n): pass"
+        the_ast = parse.parse_string(stmt, mode=1)
+        self.assertStartsWith(ast_dump(the_ast),
+            "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), "
+            "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]"
+        )
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_ternary_operator(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=expr ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        expr[asdl_stmt_seq*]: a=listcomp NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, _PyAST_Expr(a, EXTRA)) }
+        listcomp[expr_ty]: (
+            a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) }
+        )
+        for_if_clauses[asdl_comprehension_seq*]: (
+            a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
+                { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a }
+        )
+        """
+        test_source = """
+        stmt = "[i for i in a if b]"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_syntax_error_for_string(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: NAME
+        """
+        test_source = r"""
+        for text in ("a b 42 b a", "\u540d \u540d 42 \u540d \u540d"):
+            try:
+                parse.parse_string(text, mode=0)
+            except SyntaxError as e:
+                tb = traceback.format_exc()
+            self.assertTrue('File "<string>", line 1' in tb)
+            self.assertTrue(f"SyntaxError: invalid syntax" in tb)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_headers_and_trailer(self) -> None:
+        grammar_source = """
+        @header 'SOME HEADER'
+        @subheader 'SOME SUBHEADER'
+        @trailer 'SOME TRAILER'
+        start: expr+ NEWLINE? ENDMARKER
+        expr: x=NAME
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+
+        self.assertTrue("SOME HEADER" in parser_source)
+        self.assertTrue("SOME SUBHEADER" in parser_source)
+        self.assertTrue("SOME TRAILER" in parser_source)
+
+    def test_error_in_rules(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: NAME {PyTuple_New(-1)}
+        """
+        # PyTuple_New raises SystemError if an invalid argument was passed.
+        test_source = """
+        with self.assertRaises(SystemError):
+            parse.parse_string("a", mode=0)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_no_soft_keywords(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: 'foo'
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+        assert "expect_soft_keyword" not in parser_source
+
+    def test_soft_keywords(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: "foo"
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+        assert "expect_soft_keyword" in parser_source
+
+    def test_soft_keywords_parse(self) -> None:
+        grammar_source = """
+        start: "if" expr '+' expr NEWLINE
+        expr: NAME
+        """
+        test_source = """
+        valid_cases = ["if if + if"]
+        invalid_cases = ["if if"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_soft_keywords_lookahead(self) -> None:
+        grammar_source = """
+        start: &"if" "if" expr '+' expr NEWLINE
+        expr: NAME
+        """
+        test_source = """
+        valid_cases = ["if if + if"]
+        invalid_cases = ["if if"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_forced(self) -> None:
+        grammar_source = """
+        start: NAME &&':' | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected ':'", str(e.exception))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_forced_with_group(self) -> None:
+        grammar_source = """
+        start: NAME &&(':' | ';') | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        self.assertEqual(parse.parse_string("number ;", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected (':' | ';')", e.exception.args[0])
+        """
+        self.run_test(grammar_source, test_source)

Lib/test/test_peg_generator/test_first_sets.py

@@ -0,0 +1,286 @@
+import unittest
+
+from test import test_tools
+from typing import Dict, Set
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.testutil import parse_string
+    from pegen.first_sets import FirstSetCalculator
+    from pegen.grammar import Grammar
+
+
+class TestFirstSets(unittest.TestCase):
+    def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]:
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        return FirstSetCalculator(grammar.rules).calculate()
+
+    def test_alternatives(self) -> None:
+        grammar = """
+            start: expr NEWLINE? ENDMARKER
+            expr: A | B
+            A: 'a' | '-'
+            B: 'b' | '+'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "A": {"'a'", "'-'"},
+                "B": {"'+'", "'b'"},
+                "expr": {"'+'", "'a'", "'b'", "'-'"},
+                "start": {"'+'", "'a'", "'b'", "'-'"},
+            },
+        )
+
+    def test_optionals(self) -> None:
+        grammar = """
+            start: expr NEWLINE
+            expr: ['a'] ['b'] 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'c'", "'a'", "'b'"},
+                "start": {"'c'", "'a'", "'b'"},
+            },
+        )
+
+    def test_repeat_with_separator(self) -> None:
+        grammar = """
+        start: ','.thing+ NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_optional_operator(self) -> None:
+        grammar = """
+        start: sum NEWLINE
+        sum: (term)? 'b'
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"NUMBER", "'b'"},
+                "start": {"'b'", "NUMBER"},
+            },
+        )
+
+    def test_optional_literal(self) -> None:
+        grammar = """
+        start: sum NEWLINE
+        sum: '+' ? term
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"'+'", "NUMBER"},
+                "start": {"'+'", "NUMBER"},
+            },
+        )
+
+    def test_optional_after(self) -> None:
+        grammar = """
+        start: term NEWLINE
+        term: NUMBER ['+']
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_optional_before(self) -> None:
+        grammar = """
+        start: term NEWLINE
+        term: ['+'] NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}},
+        )
+
+    def test_repeat_0(self) -> None:
+        grammar = """
+        start: thing* "+" NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}},
+        )
+
+    def test_repeat_0_with_group(self) -> None:
+        grammar = """
+        start: ('+' '-')* term NEWLINE
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}},
+        )
+
+    def test_repeat_1(self) -> None:
+        grammar = """
+        start: thing+ '-' NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_repeat_1_with_group(self) -> None:
+        grammar = """
+        start: ('+' term)+ term NEWLINE
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}}
+        )
+
+    def test_gather(self) -> None:
+        grammar = """
+        start: ','.thing+ NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_positive_lookahead(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: &'a' opt
+        opt: 'a' | 'b' | 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'a'"},
+                "start": {"'a'"},
+                "opt": {"'b'", "'c'", "'a'"},
+            },
+        )
+
+    def test_negative_lookahead(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: !'a' opt
+        opt: 'a' | 'b' | 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "opt": {"'b'", "'a'", "'c'"},
+                "expr": {"'b'", "'c'"},
+                "start": {"'b'", "'c'"},
+            },
+        )
+
+    def test_left_recursion(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: ('-' term | expr '+' term | term)
+        term: NUMBER
+        foo: 'foo'
+        bar: 'bar'
+        baz: 'baz'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"NUMBER", "'-'"},
+                "term": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+                "foo": {"'foo'"},
+                "bar": {"'bar'"},
+                "baz": {"'baz'"},
+            },
+        )
+
+    def test_advance_left_recursion(self) -> None:
+        grammar = """
+        start: NUMBER | sign start
+        sign: ['-']
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}},
+        )
+
+    def test_mutual_left_recursion(self) -> None:
+        grammar = """
+        start: foo 'E'
+        foo: bar 'A' | 'B'
+        bar: foo 'C' | 'D'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "foo": {"'D'", "'B'"},
+                "bar": {"'D'"},
+                "start": {"'D'", "'B'"},
+            },
+        )
+
+    def test_nasty_left_recursion(self) -> None:
+        # TODO: Validate this
+        grammar = """
+        start: target '='
+        target: maybe '+' | NAME
+        maybe: maybe '-' | target
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
+        )
+
+    def test_nullable_rule(self) -> None:
+        grammar = """
+        start: sign thing $
+        sign: ['-']
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+            },
+        )
+
+    def test_epsilon_production_in_start_rule(self) -> None:
+        grammar = """
+        start: ['-'] $
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}}
+        )
+
+    def test_multiple_nullable_rules(self) -> None:
+        grammar = """
+        start: sign thing other another $
+        sign: ['-']
+        thing: ['+']
+        other: '*'
+        another: '/'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"'+'", ""},
+                "start": {"'+'", "'-'", "'*'"},
+                "other": {"'*'"},
+                "another": {"'/'"},
+            },
+        )

Lib/test/test_peg_generator/test_grammar_validator.py

@@ -0,0 +1,77 @@
+import unittest
+from test import test_tools
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.validator import SubRuleValidator, ValidationError
+    from pegen.validator import RaiseRuleValidator, CutValidator
+    from pegen.testutil import parse_string
+    from pegen.grammar import Grammar
+
+
+class TestPegen(unittest.TestCase):
+    def test_rule_with_no_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '-' NAME
+            | NAME '+' NAME
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        for rule_name, rule in grammar.rules.items():
+            validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_simple_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_collision_after_some_other_rules(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '*' NAME ';'
+            | NAME '-' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)
+
+    def test_raising_valid_rule(self) -> None:
+        grammar_source = """
+        start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") }
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = RaiseRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)
+
+    def test_cut_validator(self) -> None:
+        grammar_source = """
+        star: (OP ~ OP)*
+        plus: (OP ~ OP)+
+        bracket: [OP ~ OP]
+        gather: OP.(OP ~ OP)+
+        nested: [OP | NAME ~ OP]
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = CutValidator(grammar)
+        for rule_name, rule in grammar.rules.items():
+            with self.subTest(rule_name):
+                with self.assertRaises(ValidationError):
+                    validator.validate_rule(rule_name, rule)

Lib/test/test_tools/i18n_data/ascii-escapes.pot

@@ -15,30 +15,36 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
+#. Special characters that are always escaped in the POT file
 #: escapes.py:5
 msgid ""
 "\"\t\n"
 "\r\\"
 msgstr ""
 
+#. All ascii characters 0-31
 #: escapes.py:8
 msgid ""
 "\000\001\002\003\004\005\006\007\010\t\n"
 "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
 msgstr ""
 
+#. All ascii characters 32-126
 #: escapes.py:13
 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
 msgstr ""
 
+#. ascii char 127
 #: escapes.py:17
 msgid "\177"
 msgstr ""
 
+#. some characters in the 128-255 range
 #: escapes.py:20
-msgid "€   ÿ"
+msgid "\302\200 \302\240 ÿ"
 msgstr ""
 
+#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
 #: escapes.py:23
 msgid "α ㄱ 𓂀"
 msgstr ""

Lib/test/test_tools/i18n_data/comments.pot

@@ -0,0 +1,110 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: comments.py:4
+msgid "foo"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:7
+msgid "bar"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#: comments.py:11
+msgid "baz"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. with multiple
+#. lines
+#: comments.py:16
+msgid "qux"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:21
+msgid "quux"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. with multiple lines
+#. i18n: This is another translator comment
+#. with multiple lines
+#: comments.py:27
+msgid "corge"
+msgstr ""
+
+#: comments.py:31
+msgid "grault"
+msgstr ""
+
+#. i18n: This is another translator comment
+#: comments.py:36
+msgid "garply"
+msgstr ""
+
+#: comments.py:40
+msgid "george"
+msgstr ""
+
+#. i18n: This is another translator comment
+#: comments.py:45
+msgid "waldo"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is also a translator comment
+#. i18n: This is another translator comment
+#: comments.py:50
+msgid "waldo2"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#. i18n: This is yet another translator comment
+#. i18n: This is a translator comment
+#. with multiple lines
+#: comments.py:53 comments.py:56 comments.py:59 comments.py:63
+msgid "fred"
+msgstr ""
+
+#: comments.py:65
+msgid "plugh"
+msgstr ""
+
+#: comments.py:67
+msgid "foobar"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:71
+msgid "xyzzy"
+msgstr ""
+
+#: comments.py:72
+msgid "thud"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#. i18n: This is yet another translator comment
+#: comments.py:78
+msgid "foos"
+msgstr ""
+

Lib/test/test_tools/i18n_data/comments.py

@@ -0,0 +1,78 @@
+from gettext import gettext as _
+
+# Not a translator comment
+_('foo')
+
+# i18n: This is a translator comment
+_('bar')
+
+# i18n: This is a translator comment
+# i18n: This is another translator comment
+_('baz')
+
+# i18n: This is a translator comment
+# with multiple
+# lines
+_('qux')
+
+# This comment should not be included because
+# it does not start with the prefix
+# i18n: This is a translator comment
+_('quux')
+
+# i18n: This is a translator comment
+# with multiple lines
+# i18n: This is another translator comment
+# with multiple lines
+_('corge')
+
+# i18n: This comment should be ignored
+
+_('grault')
+
+# i18n: This comment should be ignored
+
+# i18n: This is another translator comment
+_('garply')
+
+# i18n: comment should be ignored
+x = 1
+_('george')
+
+# i18n: This comment should be ignored
+x = 1
+# i18n: This is another translator comment
+_('waldo')
+
+# i18n: This is a translator comment
+x = 1  # i18n: This is also a translator comment
+# i18n: This is another translator comment
+_('waldo2')
+
+# i18n: This is a translator comment
+_('fred')
+
+# i18n: This is another translator comment
+_('fred')
+
+# i18n: This is yet another translator comment
+_('fred')
+
+# i18n: This is a translator comment
+# with multiple lines
+_('fred')
+
+_('plugh')  # i18n: This comment should be ignored
+
+_('foo'  # i18n: This comment should be ignored
+  'bar')  # i18n: This comment should be ignored
+
+# i18n: This is a translator comment
+_('xyzzy')
+_('thud')
+
+
+## i18n: This is a translator comment
+# # i18n: This is another translator comment
+### ###    i18n: This is yet another translator comment
+_('foos')

Lib/test/test_tools/i18n_data/custom_keywords.pot

@@ -0,0 +1,51 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: custom_keywords.py:10 custom_keywords.py:11
+msgid "bar"
+msgstr ""
+
+#: custom_keywords.py:13
+msgid "cat"
+msgid_plural "cats"
+msgstr[0] ""
+msgstr[1] ""
+
+#: custom_keywords.py:14
+msgid "dog"
+msgid_plural "dogs"
+msgstr[0] ""
+msgstr[1] ""
+
+#: custom_keywords.py:16
+msgctxt "context"
+msgid "bar"
+msgstr ""
+
+#: custom_keywords.py:18
+msgctxt "context"
+msgid "cat"
+msgid_plural "cats"
+msgstr[0] ""
+msgstr[1] ""
+
+#: custom_keywords.py:34
+msgid "overridden"
+msgid_plural "default"
+msgstr[0] ""
+msgstr[1] ""
+

Lib/test/test_tools/i18n_data/custom_keywords.py

@@ -0,0 +1,34 @@
+from gettext import (
+    gettext as foo,
+    ngettext as nfoo,
+    pgettext as pfoo,
+    npgettext as npfoo,
+    gettext as bar,
+    gettext as _,
+)
+
+foo('bar')
+foo('bar', 'baz')
+
+nfoo('cat', 'cats', 1)
+nfoo('dog', 'dogs')
+
+pfoo('context', 'bar')
+
+npfoo('context', 'cat', 'cats', 1)
+
+# This is an unknown keyword and should be ignored
+bar('baz')
+
+# 'nfoo' requires at least 2 arguments
+nfoo('dog')
+
+# 'pfoo' requires at least 2 arguments
+pfoo('context')
+
+# 'npfoo' requires at least 3 arguments
+npfoo('context')
+npfoo('context', 'cat')
+
+# --keyword should override the default keyword
+_('overridden', 'default')

Lib/test/test_tools/i18n_data/docstrings.pot

@@ -15,26 +15,40 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
-#: docstrings.py:7
+#: docstrings.py:1
+#, docstring
+msgid "Module docstring"
+msgstr ""
+
+#: docstrings.py:9
 #, docstring
 msgid ""
 msgstr ""
 
-#: docstrings.py:18
+#: docstrings.py:15
+#, docstring
+msgid "docstring"
+msgstr ""
+
+#: docstrings.py:20
 #, docstring
 msgid ""
 "multiline\n"
-"    docstring\n"
-"    "
+"docstring"
 msgstr ""
 
-#: docstrings.py:25
+#: docstrings.py:27
 #, docstring
 msgid "docstring1"
 msgstr ""
 
-#: docstrings.py:30
+#: docstrings.py:38
 #, docstring
-msgid "Hello, {}!"
+msgid "nested docstring"
+msgstr ""
+
+#: docstrings.py:43
+#, docstring
+msgid "nested class docstring"
 msgstr ""
 

Lib/test/test_tools/i18n_data/docstrings.py

@@ -1,3 +1,5 @@
+"""Module docstring"""
+
 # Test docstring extraction
 from gettext import gettext as _
 
@@ -10,10 +12,10 @@ def test(x):
 # Leading empty line
 def test2(x):
 
-    """docstring"""  # XXX This should be extracted but isn't.
+    """docstring"""
 
 
-# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`.
+# Multiline docstrings are cleaned with `inspect.cleandoc`.
 def test3(x):
     """multiline
     docstring
@@ -27,15 +29,15 @@ def test4(x):
 
 
 def test5(x):
-    """Hello, {}!""".format("world!")  # XXX This should not be extracted.
+    """Hello, {}!""".format("world!")  # This should not be extracted.
 
 
 # Nested docstrings
 def test6(x):
     def inner(y):
-        """nested docstring"""  # XXX This should be extracted but isn't.
+        """nested docstring"""
 
 
 class Outer:
     class Inner:
-        "nested class docstring"  # XXX This should be extracted but isn't.
+        "nested class docstring"

Lib/test/test_tools/i18n_data/escapes.pot

@@ -15,30 +15,36 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
+#. Special characters that are always escaped in the POT file
 #: escapes.py:5
 msgid ""
 "\"\t\n"
 "\r\\"
 msgstr ""
 
+#. All ascii characters 0-31
 #: escapes.py:8
 msgid ""
 "\000\001\002\003\004\005\006\007\010\t\n"
 "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
 msgstr ""
 
+#. All ascii characters 32-126
 #: escapes.py:13
 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
 msgstr ""
 
+#. ascii char 127
 #: escapes.py:17
 msgid "\177"
 msgstr ""
 
+#. some characters in the 128-255 range
 #: escapes.py:20
 msgid "\302\200 \302\240 \303\277"
 msgstr ""
 
+#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
 #: escapes.py:23
 msgid "\316\261 \343\204\261 \360\223\202\200"
 msgstr ""

Lib/test/test_tools/i18n_data/messages.pot

@@ -15,53 +15,85 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
-#: messages.py:5
+#: messages.py:16
 msgid ""
 msgstr ""
 
-#: messages.py:8 messages.py:9
+#: messages.py:19 messages.py:20 messages.py:21
 msgid "parentheses"
 msgstr ""
 
-#: messages.py:12
+#: messages.py:24
 msgid "Hello, world!"
 msgstr ""
 
-#: messages.py:15
+#: messages.py:27
 msgid ""
 "Hello,\n"
 "    multiline!\n"
 msgstr ""
 
-#: messages.py:29
+#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94
+#: messages.py:99 messages.py:100 messages.py:101
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:47
+msgid "something"
+msgstr ""
+
+#: messages.py:50
 msgid "Hello, {}!"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "1"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "2"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "A"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "B"
 msgstr ""
 
-#: messages.py:36
+#: messages.py:57
 msgid "set"
 msgstr ""
 
-#: messages.py:42
+#: messages.py:62 messages.py:63
 msgid "nested string"
 msgstr ""
 
-#: messages.py:47
+#: messages.py:68
 msgid "baz"
 msgstr ""
 
+#: messages.py:71 messages.py:75
+msgid "default value"
+msgstr ""
+
+#: messages.py:91 messages.py:92 messages.py:95 messages.py:96
+msgctxt "context"
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:102
+msgid "domain foo"
+msgstr ""
+
+#: messages.py:118 messages.py:119
+msgid "world"
+msgid_plural "worlds"
+msgstr[0] ""
+msgstr[1] ""
+

Lib/test/test_tools/i18n_data/messages.py

@@ -1,5 +1,16 @@
 # Test message extraction
-from gettext import gettext as _
+from gettext import (
+    gettext,
+    ngettext,
+    pgettext,
+    npgettext,
+    dgettext,
+    dngettext,
+    dpgettext,
+    dnpgettext
+)
+
+_ = gettext
 
 # Empty string
 _("")
@@ -7,6 +18,7 @@ _("")
 # Extra parentheses
 (_("parentheses"))
 ((_("parentheses")))
+_(("parentheses"))
 
 # Multiline strings
 _("Hello, "
@@ -21,13 +33,22 @@ _()
 _(None)
 _(1)
 _(False)
-_(x="kwargs are not allowed")
+_(["invalid"])
+_({"invalid"})
+_("string"[3])
+_("string"[:3])
+_({"string": "foo"})
+
+# pygettext does not allow keyword arguments, but both xgettext and pybabel do
+_(x="kwargs are not allowed!")
+
+# Unusual, but valid arguments
 _("foo", "bar")
 _("something", x="something else")
 
 # .format()
 _("Hello, {}!").format("world")  # valid
-_("Hello, {}!".format("world"))  # invalid
+_("Hello, {}!".format("world"))  # invalid, but xgettext extracts the first string
 
 # Nested structures
 _("1"), _("2")
@@ -38,7 +59,7 @@ obj = {'a': _("A"), 'b': _("B")}
 
 # Nested functions and classes
 def test():
-    _("nested string")  # XXX This should be extracted but isn't.
+    _("nested string")
     [_("nested string")]
 
 
@@ -47,11 +68,11 @@ class Foo:
         return _("baz")
 
 
-def bar(x=_('default value')):  # XXX This should be extracted but isn't.
+def bar(x=_('default value')):
     pass
 
 
-def baz(x=[_('default value')]):  # XXX This should be extracted but isn't.
+def baz(x=[_('default value')]):
     pass
 
 
@@ -62,3 +83,37 @@ def _(x):
 
 def _(x="don't extract me"):
     pass
+
+
+# Other gettext functions
+gettext("foo")
+ngettext("foo", "foos", 1)
+pgettext("context", "foo")
+npgettext("context", "foo", "foos", 1)
+dgettext("domain", "foo")
+dngettext("domain", "foo", "foos", 1)
+dpgettext("domain", "context", "foo")
+dnpgettext("domain", "context", "foo", "foos", 1)
+
+# Complex arguments
+ngettext("foo", "foos", 42 + (10 - 20))
+ngettext("foo", "foos", *args)
+ngettext("foo", "foos", **kwargs)
+dgettext(["some", {"complex"}, ("argument",)], "domain foo")
+
+# Invalid calls which are not extracted
+gettext()
+ngettext('foo')
+pgettext('context')
+npgettext('context', 'foo')
+dgettext('domain')
+dngettext('domain', 'foo')
+dpgettext('domain', 'context')
+dnpgettext('domain', 'context', 'foo')
+dgettext(*args, 'foo')
+dpgettext(*args, 'context', 'foo')
+dnpgettext(*args, 'context', 'foo', 'foos')
+
+# f-strings
+f"Hello, {_('world')}!"
+f"Hello, {ngettext('world', 'worlds', 3)}!"

Lib/test/test_tools/i18n_data/multiple_keywords.pot

@@ -0,0 +1,38 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: multiple_keywords.py:3
+msgid "bar"
+msgstr ""
+
+#: multiple_keywords.py:5
+msgctxt "baz"
+msgid "qux"
+msgstr ""
+
+#: multiple_keywords.py:9
+msgctxt "corge"
+msgid "grault"
+msgstr ""
+
+#: multiple_keywords.py:11
+msgctxt "xyzzy"
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+

Lib/test/test_tools/i18n_data/multiple_keywords.py

@@ -0,0 +1,11 @@
+from gettext import gettext as foo
+
+foo('bar')
+
+foo('baz', 'qux')
+
+# The 't' specifier is not supported, so the following
+# call is extracted as pgettext instead of ngettext.
+foo('corge', 'grault', 1)
+
+foo('xyzzy', 'foo', 'foos', 1)

Lib/test/test_tools/test_compute_changes.py

@@ -0,0 +1,144 @@
+"""Tests to cover the Tools/build/compute-changes.py script."""
+
+import importlib
+import os
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from test.test_tools import skip_if_missing, imports_under_tool
+
+skip_if_missing("build")
+
+with patch.dict(os.environ, {"GITHUB_DEFAULT_BRANCH": "main"}):
+    with imports_under_tool("build"):
+        compute_changes = importlib.import_module("compute-changes")
+
+process_changed_files = compute_changes.process_changed_files
+Outputs = compute_changes.Outputs
+ANDROID_DIRS = compute_changes.ANDROID_DIRS
+IOS_DIRS = compute_changes.IOS_DIRS
+MACOS_DIRS = compute_changes.MACOS_DIRS
+WASI_DIRS = compute_changes.WASI_DIRS
+RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE
+UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES
+LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS
+
+
+class TestProcessChangedFiles(unittest.TestCase):
+
+    def test_windows(self):
+        f = {Path(".github/workflows/reusable-windows.yml")}
+        result = process_changed_files(f)
+        self.assertTrue(result.run_tests)
+        self.assertTrue(result.run_windows_tests)
+
+    def test_docs(self):
+        for f in (
+            ".github/workflows/reusable-docs.yml",
+            "Doc/library/datetime.rst",
+            "Doc/Makefile",
+        ):
+            with self.subTest(f=f):
+                result = process_changed_files({Path(f)})
+                self.assertTrue(result.run_docs)
+                self.assertFalse(result.run_tests)
+
+    def test_ci_fuzz_stdlib(self):
+        for p in LIBRARY_FUZZER_PATHS:
+            with self.subTest(p=p):
+                if p.is_dir():
+                    f = p / "file"
+                elif p.is_file():
+                    f = p
+                else:
+                    continue
+                result = process_changed_files({f})
+                self.assertTrue(result.run_ci_fuzz_stdlib)
+
+    def test_android(self):
+        for d in ANDROID_DIRS:
+            with self.subTest(d=d):
+                result = process_changed_files({Path(d) / "file"})
+                self.assertTrue(result.run_tests)
+                self.assertTrue(result.run_android)
+                self.assertFalse(result.run_windows_tests)
+
+    def test_ios(self):
+        for d in IOS_DIRS:
+            with self.subTest(d=d):
+                result = process_changed_files({Path(d) / "file"})
+                self.assertTrue(result.run_tests)
+                self.assertTrue(result.run_ios)
+                self.assertFalse(result.run_windows_tests)
+
+    def test_macos(self):
+        f = {Path(".github/workflows/reusable-macos.yml")}
+        result = process_changed_files(f)
+        self.assertTrue(result.run_tests)
+        self.assertTrue(result.run_macos)
+
+        for d in MACOS_DIRS:
+            with self.subTest(d=d):
+                result = process_changed_files({Path(d) / "file"})
+                self.assertTrue(result.run_tests)
+                self.assertTrue(result.run_macos)
+                self.assertFalse(result.run_windows_tests)
+
+    def test_wasi(self):
+        f = {Path(".github/workflows/reusable-wasi.yml")}
+        result = process_changed_files(f)
+        self.assertTrue(result.run_tests)
+        self.assertTrue(result.run_wasi)
+
+        for d in WASI_DIRS:
+            with self.subTest(d=d):
+                result = process_changed_files({d / "file"})
+                self.assertTrue(result.run_tests)
+                self.assertTrue(result.run_wasi)
+                self.assertFalse(result.run_windows_tests)
+
+    def test_unix(self):
+        for f in UNIX_BUILD_SYSTEM_FILE_NAMES:
+            with self.subTest(f=f):
+                result = process_changed_files({f})
+                self.assertTrue(result.run_tests)
+                self.assertFalse(result.run_windows_tests)
+
+    def test_msi(self):
+        for f in (
+            ".github/workflows/reusable-windows-msi.yml",
+            "Tools/msi/build.bat",
+        ):
+            with self.subTest(f=f):
+                result = process_changed_files({Path(f)})
+                self.assertTrue(result.run_windows_msi)
+
+    def test_all_run(self):
+        for f in (
+            ".github/workflows/some-new-workflow.yml",
+            ".github/workflows/build.yml",
+        ):
+            with self.subTest(f=f):
+                result = process_changed_files({Path(f)})
+                self.assertTrue(result.run_tests)
+                self.assertTrue(result.run_android)
+                self.assertTrue(result.run_ios)
+                self.assertTrue(result.run_macos)
+                self.assertTrue(result.run_ubuntu)
+                self.assertTrue(result.run_wasi)
+
+    def test_all_ignored(self):
+        for f in RUN_TESTS_IGNORE:
+            with self.subTest(f=f):
+                self.assertEqual(process_changed_files({Path(f)}), Outputs())
+
+    def test_wasi_and_android(self):
+        f = {Path(".github/workflows/reusable-wasi.yml"), Path("Android/file")}
+        result = process_changed_files(f)
+        self.assertTrue(result.run_tests)
+        self.assertTrue(result.run_wasi)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_tools/test_i18n.py

@@ -8,7 +8,7 @@ from textwrap import dedent
 from pathlib import Path
 
 from test.support.script_helper import assert_python_ok
-from test.test_tools import skip_if_missing, toolsdir
+from test.test_tools import imports_under_tool, skip_if_missing, toolsdir
 from test.support.os_helper import temp_cwd, temp_dir
 
 
@@ -17,6 +17,11 @@ skip_if_missing()
 DATA_DIR = Path(__file__).resolve().parent / 'i18n_data'
 
 
+with imports_under_tool("i18n"):
+    from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS,
+                           unparse_spec)
+
+
 def normalize_POT_file(pot):
     """Normalize the POT creation timestamp, charset and
     file locations to make the POT file easier to compare.
@@ -87,7 +92,8 @@ class Test_pygettext(unittest.TestCase):
         self.maxDiff = None
         self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
 
-    def extract_from_str(self, module_content, *, args=(), strict=True):
+    def extract_from_str(self, module_content, *, args=(), strict=True,
+                         with_stderr=False, raw=False):
         """Return all msgids extracted from module_content."""
         filename = 'test.py'
         with temp_cwd(None):
@@ -98,12 +104,19 @@ class Test_pygettext(unittest.TestCase):
                 self.assertEqual(res.err, b'')
             with open('messages.pot', encoding='utf-8') as fp:
                 data = fp.read()
-        return self.get_msgids(data)
+        if not raw:
+            data = self.get_msgids(data)
+        if not with_stderr:
+            return data
+        return data, res.err
 
     def extract_docstrings_from_str(self, module_content):
         """Return all docstrings extracted from module_content."""
         return self.extract_from_str(module_content, args=('--docstrings',), strict=False)
 
+    def get_stderr(self, module_content):
+        return self.extract_from_str(module_content, strict=False, with_stderr=True)[1]
+
     def test_header(self):
         """Make sure the required fields are in the header, according to:
            http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
@@ -149,6 +162,14 @@ class Test_pygettext(unittest.TestCase):
             # This will raise if the date format does not exactly match.
             datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
 
+    def test_output_option(self):
+        for opt in ('-o', '--output='):
+            with temp_cwd():
+                assert_python_ok(self.script, f'{opt}test')
+                self.assertTrue(os.path.exists('test'))
+                res = assert_python_ok(self.script, f'{opt}-')
+                self.assertIn(b'Project-Id-Version: PACKAGE VERSION', res.out)
+
     def test_funcdocstring(self):
         for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
             with self.subTest(doc):
@@ -332,14 +353,14 @@ class Test_pygettext(unittest.TestCase):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', 'bar')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
 
     def test_calls_in_fstring_with_keyword_args(self):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', bar='baz')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
         self.assertNotIn('baz', msgids)
 
@@ -400,17 +421,195 @@ class Test_pygettext(unittest.TestCase):
             self.assertIn(f'msgid "{text2}"', data)
             self.assertNotIn(text3, data)
 
+    def test_help_text(self):
+        """Test that the help text is displayed."""
+        res = assert_python_ok(self.script, '--help')
+        self.assertEqual(res.out, b'')
+        self.assertIn(b'pygettext -- Python equivalent of xgettext(1)', res.err)
+
+    def test_error_messages(self):
+        """Test that pygettext outputs error messages to stderr."""
+        stderr = self.get_stderr(dedent('''\
+        _(1+2)
+        ngettext('foo')
+        dgettext(*args, 'foo')
+        '''))
+
+        # Normalize line endings on Windows
+        stderr = stderr.decode('utf-8').replace('\r', '')
+
+        self.assertEqual(
+            stderr,
+            "*** test.py:1: Expected a string constant for argument 1, got 1 + 2\n"
+            "*** test.py:2: Expected at least 2 positional argument(s) in gettext call, got 1\n"
+            "*** test.py:3: Variable positional arguments are not allowed in gettext calls\n"
+        )
+
+    def test_extract_all_comments(self):
+        """
+        Test that the --add-comments option without an
+        explicit tag extracts all translator comments.
+        """
+        for arg in ('--add-comments', '-c'):
+            with self.subTest(arg=arg):
+                data = self.extract_from_str(dedent('''\
+                # Translator comment
+                _("foo")
+                '''), args=(arg,), raw=True)
+                self.assertIn('#. Translator comment', data)
+
+    def test_comments_with_multiple_tags(self):
+        """
+        Test that multiple --add-comments tags can be specified.
+        """
+        for arg in ('--add-comments={}', '-c{}'):
+            with self.subTest(arg=arg):
+                args = (arg.format('foo:'), arg.format('bar:'))
+                data = self.extract_from_str(dedent('''\
+                # foo: comment
+                _("foo")
+
+                # bar: comment
+                _("bar")
+
+                # baz: comment
+                _("baz")
+                '''), args=args, raw=True)
+                self.assertIn('#. foo: comment', data)
+                self.assertIn('#. bar: comment', data)
+                self.assertNotIn('#. baz: comment', data)
+
+    def test_comments_not_extracted_without_tags(self):
+        """
+        Test that translator comments are not extracted without
+        specifying --add-comments.
+        """
+        data = self.extract_from_str(dedent('''\
+        # Translator comment
+        _("foo")
+        '''), raw=True)
+        self.assertNotIn('#.', data)
+
+    def test_parse_keyword_spec(self):
+        valid = (
+            ('foo', ('foo', {'msgid': 0})),
+            ('foo:1', ('foo', {'msgid': 0})),
+            ('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
+            ('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
+            ('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
+            ('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
+            ('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
+        )
+        for spec, expected in valid:
+            with self.subTest(spec=spec):
+                self.assertEqual(parse_spec(spec), expected)
+                # test unparse-parse round-trip
+                self.assertEqual(parse_spec(unparse_spec(*expected)), expected)
+
+        invalid = (
+            ('foo:', "Invalid keyword spec 'foo:': missing argument positions"),
+            ('foo:bar', "Invalid keyword spec 'foo:bar': position is not an integer"),
+            ('foo:0', "Invalid keyword spec 'foo:0': argument positions must be strictly positive"),
+            ('foo:-2', "Invalid keyword spec 'foo:-2': argument positions must be strictly positive"),
+            ('foo:1,1', "Invalid keyword spec 'foo:1,1': duplicate positions"),
+            ('foo:1,2,1', "Invalid keyword spec 'foo:1,2,1': duplicate positions"),
+            ('foo:1c,2,1c', "Invalid keyword spec 'foo:1c,2,1c': duplicate positions"),
+            ('foo:1c,2,3c', "Invalid keyword spec 'foo:1c,2,3c': msgctxt can only appear once"),
+            ('foo:1,2,3', "Invalid keyword spec 'foo:1,2,3': too many positions"),
+            ('foo:1c', "Invalid keyword spec 'foo:1c': msgctxt cannot appear without msgid"),
+        )
+        for spec, message in invalid:
+            with self.subTest(spec=spec):
+                with self.assertRaises(ValueError) as cm:
+                    parse_spec(spec)
+                self.assertEqual(str(cm.exception), message)
+
+    def test_process_keywords(self):
+        default_keywords = {name: [spec] for name, spec
+                            in DEFAULTKEYWORDS.items()}
+        inputs = (
+            (['foo'], True),
+            (['_:1,2'], True),
+            (['foo', 'foo:1,2'], True),
+            (['foo'], False),
+            (['_:1,2', '_:1c,2,3', 'pgettext'], False),
+            # Duplicate entries
+            (['foo', 'foo'], True),
+            (['_'], False)
+        )
+        expected = (
+            {'foo': [{'msgid': 0}]},
+            {'_': [{'msgid': 0, 'msgid_plural': 1}]},
+            {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]},
+            default_keywords | {'foo': [{'msgid': 0}]},
+            default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1},
+                                      {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
+                                      {'msgid': 0}],
+                                'pgettext': [{'msgid': 0},
+                                             {'msgctxt': 0, 'msgid': 1}]},
+            {'foo': [{'msgid': 0}]},
+            default_keywords,
+        )
+        for (keywords, no_default_keywords), expected in zip(inputs, expected):
+            with self.subTest(keywords=keywords,
+                              no_default_keywords=no_default_keywords):
+                processed = process_keywords(
+                    keywords,
+                    no_default_keywords=no_default_keywords)
+                self.assertEqual(processed, expected)
+
+    def test_multiple_keywords_same_funcname_errors(self):
+        # If at least one keyword spec for a given funcname matches,
+        # no error should be printed.
+        msgids, stderr = self.extract_from_str(dedent('''\
+        _("foo", 42)
+        _(42, "bar")
+        '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True)
+        self.assertIn('foo', msgids)
+        self.assertIn('bar', msgids)
+        self.assertEqual(stderr, b'')
+
+        # If no keyword spec for a given funcname matches,
+        # all errors are printed.
+        msgids, stderr = self.extract_from_str(dedent('''\
+        _(x, 42)
+        _(42, y)
+        '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True,
+              strict=False)
+        self.assertEqual(msgids, [''])
+        # Normalize line endings on Windows
+        stderr = stderr.decode('utf-8').replace('\r', '')
+        self.assertEqual(
+            stderr,
+            '*** test.py:1: No keywords matched gettext call "_":\n'
+            '\tkeyword="_": Expected a string constant for argument 1, got x\n'
+            '\tkeyword="_:2": Expected a string constant for argument 2, got 42\n'
+            '*** test.py:2: No keywords matched gettext call "_":\n'
+            '\tkeyword="_": Expected a string constant for argument 1, got 42\n'
+            '\tkeyword="_:2": Expected a string constant for argument 2, got y\n')
+
 
 def extract_from_snapshots():
     snapshots = {
-        'messages.py': ('--docstrings',),
+        'messages.py': (),
         'fileloc.py': ('--docstrings',),
         'docstrings.py': ('--docstrings',),
+        'comments.py': ('--add-comments=i18n:',),
+        'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
+                               '--keyword=pfoo:1c,2',
+                               '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'),
+        'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2',
+                                 '--keyword=foo:1,2',
+                                 # repeat a keyword to make sure it is extracted only once
+                                 '--keyword=foo', '--keyword=foo'),
         # == Test character escaping
         # Escape ascii and unicode:
-        'escapes.py': ('--escape',),
+        'escapes.py': ('--escape', '--add-comments='),
         # Escape only ascii and let unicode pass through:
-        ('escapes.py', 'ascii-escapes.pot'): (),
+        ('escapes.py', 'ascii-escapes.pot'): ('--add-comments=',),
     }
 
     for filename, args in snapshots.items():

Lib/test/test_tools/test_makefile.py

@@ -48,15 +48,18 @@ class TestMakefile(unittest.TestCase):
             if dirname == '__pycache__' or dirname.startswith('.'):
                 dirs.clear()  # do not process subfolders
                 continue
-            # Skip empty dirs:
+
+            # Skip empty dirs (ignoring hidden files and __pycache__):
+            files = [
+                filename for filename in files
+                if not filename.startswith('.')
+            ]
+            dirs = [
+                dirname for dirname in dirs
+                if not dirname.startswith('.') and dirname != "__pycache__"
+            ]
             if not dirs and not files:
                 continue
-            # Skip dirs with hidden-only files:
-            if files and all(
-                filename.startswith('.') or filename == '__pycache__'
-                for filename in files
-            ):
-                continue
 
             relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
             with self.subTest(relpath=relpath):

Lib/test/test_tools/test_msgfmt.py

@@ -1,6 +1,7 @@
 """Tests for the Tools/i18n/msgfmt.py tool."""
 
 import json
+import struct
 import sys
 import unittest
 from gettext import GNUTranslations
@@ -8,18 +9,21 @@ from pathlib import Path
 
 from test.support.os_helper import temp_cwd
 from test.support.script_helper import assert_python_failure, assert_python_ok
-from test.test_tools import skip_if_missing, toolsdir
+from test.test_tools import imports_under_tool, skip_if_missing, toolsdir
 
 
 skip_if_missing('i18n')
 
 data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
 script_dir = Path(toolsdir) / 'i18n'
-msgfmt = script_dir / 'msgfmt.py'
+msgfmt_py = script_dir / 'msgfmt.py'
+
+with imports_under_tool("i18n"):
+    import msgfmt
 
 
 def compile_messages(po_file, mo_file):
-    assert_python_ok(msgfmt, '-o', mo_file, po_file)
+    assert_python_ok(msgfmt_py, '-o', mo_file, po_file)
 
 
 class CompilationTest(unittest.TestCase):
@@ -40,6 +44,31 @@ class CompilationTest(unittest.TestCase):
 
                     self.assertDictEqual(actual._catalog, expected._catalog)
 
+    def test_binary_header(self):
+        with temp_cwd():
+            tmp_mo_file = 'messages.mo'
+            compile_messages(data_dir / "general.po", tmp_mo_file)
+            with open(tmp_mo_file, 'rb') as f:
+                mo_data = f.read()
+
+        (
+            magic,
+            version,
+            num_strings,
+            orig_table_offset,
+            trans_table_offset,
+            hash_table_size,
+            hash_table_offset,
+        ) = struct.unpack("=7I", mo_data[:28])
+
+        self.assertEqual(magic, 0x950412de)
+        self.assertEqual(version, 0)
+        self.assertEqual(num_strings, 9)
+        self.assertEqual(orig_table_offset, 28)
+        self.assertEqual(trans_table_offset, 100)
+        self.assertEqual(hash_table_size, 0)
+        self.assertEqual(hash_table_offset, 0)
+
     def test_translations(self):
         with open(data_dir / 'general.mo', 'rb') as f:
             t = GNUTranslations(f)
@@ -62,6 +91,14 @@ class CompilationTest(unittest.TestCase):
                                      '%d emails sent.', 2),
                          '%d emails sent.')
 
+    def test_po_with_bom(self):
+        with temp_cwd():
+            Path('bom.po').write_bytes(b'\xef\xbb\xbfmsgid "Python"\nmsgstr "Pioton"\n')
+
+            res = assert_python_failure(msgfmt_py, 'bom.po')
+            err = res.err.decode('utf-8')
+            self.assertIn('The file bom.po starts with a UTF-8 BOM', err)
+
     def test_invalid_msgid_plural(self):
         with temp_cwd():
             Path('invalid.po').write_text('''\
@@ -69,7 +106,7 @@ msgid_plural "plural"
 msgstr[0] "singular"
 ''')
 
-            res = assert_python_failure(msgfmt, 'invalid.po')
+            res = assert_python_failure(msgfmt_py, 'invalid.po')
             err = res.err.decode('utf-8')
             self.assertIn('msgid_plural not preceded by msgid', err)
 
@@ -80,7 +117,7 @@ msgid "foo"
 msgstr[0] "bar"
 ''')
 
-            res = assert_python_failure(msgfmt, 'invalid.po')
+            res = assert_python_failure(msgfmt_py, 'invalid.po')
             err = res.err.decode('utf-8')
             self.assertIn('plural without msgid_plural', err)
 
@@ -92,7 +129,7 @@ msgid_plural "foos"
 msgstr "bar"
 ''')
 
-            res = assert_python_failure(msgfmt, 'invalid.po')
+            res = assert_python_failure(msgfmt_py, 'invalid.po')
             err = res.err.decode('utf-8')
             self.assertIn('indexed msgstr required for plural', err)
 
@@ -102,38 +139,136 @@ msgstr "bar"
 "foo"
 ''')
 
-            res = assert_python_failure(msgfmt, 'invalid.po')
+            res = assert_python_failure(msgfmt_py, 'invalid.po')
             err = res.err.decode('utf-8')
             self.assertIn('Syntax error', err)
 
+
+class POParserTest(unittest.TestCase):
+    @classmethod
+    def tearDownClass(cls):
+        # msgfmt uses a global variable to store messages,
+        # clear it after the tests.
+        msgfmt.MESSAGES.clear()
+
+    def test_strings(self):
+        # Test that the PO parser correctly handles and unescape
+        # strings in the PO file.
+        # The PO file format allows for a variety of escape sequences,
+        # octal and hex escapes.
+        valid_strings = (
+            # empty strings
+            ('""', ''),
+            ('"" "" ""', ''),
+            # allowed escape sequences
+            (r'"\\"', '\\'),
+            (r'"\""', '"'),
+            (r'"\t"', '\t'),
+            (r'"\n"', '\n'),
+            (r'"\r"', '\r'),
+            (r'"\f"', '\f'),
+            (r'"\a"', '\a'),
+            (r'"\b"', '\b'),
+            (r'"\v"', '\v'),
+            # non-empty strings
+            ('"foo"', 'foo'),
+            ('"foo" "bar"', 'foobar'),
+            ('"foo""bar"', 'foobar'),
+            ('"" "foo" ""', 'foo'),
+            # newlines and tabs
+            (r'"foo\nbar"', 'foo\nbar'),
+            (r'"foo\n" "bar"', 'foo\nbar'),
+            (r'"foo\tbar"', 'foo\tbar'),
+            (r'"foo\t" "bar"', 'foo\tbar'),
+            # escaped quotes
+            (r'"foo\"bar"', 'foo"bar'),
+            (r'"foo\"" "bar"', 'foo"bar'),
+            (r'"foo\\" "bar"', 'foo\\bar'),
+            # octal escapes
+            (r'"\120\171\164\150\157\156"', 'Python'),
+            (r'"\120\171\164" "\150\157\156"', 'Python'),
+            (r'"\"\120\171\164" "\150\157\156\""', '"Python"'),
+            # hex escapes
+            (r'"\x50\x79\x74\x68\x6f\x6e"', 'Python'),
+            (r'"\x50\x79\x74" "\x68\x6f\x6e"', 'Python'),
+            (r'"\"\x50\x79\x74" "\x68\x6f\x6e\""', '"Python"'),
+        )
+
+        with temp_cwd():
+            for po_string, expected in valid_strings:
+                with self.subTest(po_string=po_string):
+                    # Construct a PO file with a single entry,
+                    # compile it, read it into a catalog and
+                    # check the result.
+                    po = f'msgid {po_string}\nmsgstr "translation"'
+                    Path('messages.po').write_text(po)
+                    # Reset the global MESSAGES dictionary
+                    msgfmt.MESSAGES.clear()
+                    msgfmt.make('messages.po', 'messages.mo')
+
+                    with open('messages.mo', 'rb') as f:
+                        actual = GNUTranslations(f)
+
+                    self.assertDictEqual(actual._catalog, {expected: 'translation'})
+
+        invalid_strings = (
+            # "''",  # invalid but currently accepted
+            '"',
+            '"""',
+            '"" "',
+            'foo',
+            '"" "foo',
+            '"foo" foo',
+            '42',
+            '"" 42 ""',
+            # disallowed escape sequences
+            # r'"\'"',  # invalid but currently accepted
+            # r'"\e"',  # invalid but currently accepted
+            # r'"\8"',  # invalid but currently accepted
+            # r'"\9"',  # invalid but currently accepted
+            r'"\x"',
+            r'"\u1234"',
+            r'"\N{ROMAN NUMERAL NINE}"'
+        )
+        with temp_cwd():
+            for invalid_string in invalid_strings:
+                with self.subTest(string=invalid_string):
+                    po = f'msgid {invalid_string}\nmsgstr "translation"'
+                    Path('messages.po').write_text(po)
+                    # Reset the global MESSAGES dictionary
+                    msgfmt.MESSAGES.clear()
+                    with self.assertRaises(Exception):
+                        msgfmt.make('messages.po', 'messages.mo')
+
+
 class CLITest(unittest.TestCase):
 
     def test_help(self):
         for option in ('--help', '-h'):
-            res = assert_python_ok(msgfmt, option)
+            res = assert_python_ok(msgfmt_py, option)
             err = res.err.decode('utf-8')
             self.assertIn('Generate binary message catalog from textual translation description.', err)
 
     def test_version(self):
         for option in ('--version', '-V'):
-            res = assert_python_ok(msgfmt, option)
+            res = assert_python_ok(msgfmt_py, option)
             out = res.out.decode('utf-8').strip()
             self.assertEqual('msgfmt.py 1.2', out)
 
     def test_invalid_option(self):
-        res = assert_python_failure(msgfmt, '--invalid-option')
+        res = assert_python_failure(msgfmt_py, '--invalid-option')
         err = res.err.decode('utf-8')
         self.assertIn('Generate binary message catalog from textual translation description.', err)
         self.assertIn('option --invalid-option not recognized', err)
 
     def test_no_input_file(self):
-        res = assert_python_ok(msgfmt)
+        res = assert_python_ok(msgfmt_py)
         err = res.err.decode('utf-8').replace('\r\n', '\n')
         self.assertIn('No input file given\n'
                       "Try `msgfmt --help' for more information.", err)
 
     def test_nonexistent_file(self):
-        assert_python_failure(msgfmt, 'nonexistent.po')
+        assert_python_failure(msgfmt_py, 'nonexistent.po')
 
 
 def update_catalog_snapshots():

Lib/test/test_type_cache.py

@@ -0,0 +1,265 @@
+""" Tests for the internal type cache in CPython. """
+import dis
+import unittest
+import warnings
+from test import support
+from test.support import import_helper, requires_specialization, requires_specialization_ft
+try:
+    from sys import _clear_type_cache
+except ImportError:
+    _clear_type_cache = None
+
+# Skip this test if the _testcapi module isn't available.
+_testcapi = import_helper.import_module("_testcapi")
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
+type_get_version = _testcapi.type_get_version
+type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe
+type_assign_version = _testcapi.type_assign_version
+type_modified = _testcapi.type_modified
+
+def clear_type_cache():
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", DeprecationWarning)
+        _clear_type_cache()
+
+@support.cpython_only
+@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache")
+class TypeCacheTests(unittest.TestCase):
+    def test_tp_version_tag_unique(self):
+        """tp_version_tag should be unique assuming no overflow, even after
+        clearing type cache.
+        """
+        # Check if global version tag has already overflowed.
+        Y = type('Y', (), {})
+        Y.x = 1
+        Y.x  # Force a _PyType_Lookup, populating version tag
+        y_ver = type_get_version(Y)
+        # Overflow, or not enough left to conduct the test.
+        if y_ver == 0 or y_ver > 0xFFFFF000:
+            self.skipTest("Out of type version tags")
+        # Note: try to avoid any method lookups within this loop,
+        # It will affect global version tag.
+        all_version_tags = []
+        append_result = all_version_tags.append
+        assertNotEqual = self.assertNotEqual
+        for _ in range(30):
+            clear_type_cache()
+            X = type('Y', (), {})
+            X.x = 1
+            X.x
+            tp_version_tag_after = type_get_version(X)
+            assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed")
+            append_result(tp_version_tag_after)
+        self.assertEqual(len(set(all_version_tags)), 30,
+                         msg=f"{all_version_tags} contains non-unique versions")
+
+    def test_type_assign_version(self):
+        class C:
+            x = 5
+
+        self.assertEqual(type_assign_version(C), 1)
+        c_ver = type_get_version(C)
+
+        C.x = 6
+        self.assertEqual(type_get_version(C), 0)
+        self.assertEqual(type_assign_version(C), 1)
+        self.assertNotEqual(type_get_version(C), 0)
+        self.assertNotEqual(type_get_version(C), c_ver)
+
+    def test_type_assign_specific_version(self):
+        """meta-test for type_assign_specific_version_unsafe"""
+        class C:
+            pass
+
+        type_assign_version(C)
+        orig_version = type_get_version(C)
+        if orig_version == 0:
+            self.skipTest("Could not assign a valid type version")
+
+        type_modified(C)
+        type_assign_specific_version_unsafe(C, orig_version + 5)
+        type_assign_version(C)  # this should do nothing
+
+        new_version = type_get_version(C)
+        self.assertEqual(new_version, orig_version + 5)
+
+        clear_type_cache()
+
+    def test_per_class_limit(self):
+        class C:
+            x = 0
+
+        type_assign_version(C)
+        orig_version = type_get_version(C)
+        for i in range(1001):
+            C.x = i
+            type_assign_version(C)
+
+        new_version = type_get_version(C)
+        self.assertEqual(new_version, 0)
+
+    def test_119462(self):
+
+        class Holder:
+            value = None
+
+            @classmethod
+            def set_value(cls):
+                cls.value = object()
+
+        class HolderSub(Holder):
+            pass
+
+        for _ in range(1050):
+            Holder.set_value()
+            HolderSub.value
+
+@support.cpython_only
+class TypeCacheWithSpecializationTests(unittest.TestCase):
+    def tearDown(self):
+        clear_type_cache()
+
+    def _assign_valid_version_or_skip(self, type_):
+        type_modified(type_)
+        type_assign_version(type_)
+        if type_get_version(type_) == 0:
+            self.skipTest("Could not assign valid type version")
+
+    def _no_more_versions(self, user_type):
+        type_modified(user_type)
+        for _ in range(1001):
+            type_assign_specific_version_unsafe(user_type, 1000_000_000)
+        type_assign_specific_version_unsafe(user_type, 0)
+        self.assertEqual(type_get_version(user_type), 0)
+
+    def _all_opnames(self, func):
+        return set(instr.opname for instr in dis.Bytecode(func, adaptive=True))
+
+    def _check_specialization(self, func, arg, opname, *, should_specialize):
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
+            func(arg)
+
+        if should_specialize:
+            self.assertNotIn(opname, self._all_opnames(func))
+        else:
+            self.assertIn(opname, self._all_opnames(func))
+
+    @requires_specialization
+    def test_class_load_attr_specialization_user_type(self):
+        class A:
+            def foo(self):
+                pass
+
+        self._assign_valid_version_or_skip(A)
+
+        def load_foo_1(type_):
+            type_.foo
+
+        self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True)
+        del load_foo_1
+
+        self._no_more_versions(A)
+
+        def load_foo_2(type_):
+            return type_.foo
+
+        self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False)
+
+    @requires_specialization
+    def test_class_load_attr_specialization_static_type(self):
+        self.assertNotEqual(type_get_version(str), 0)
+        self.assertNotEqual(type_get_version(bytes), 0)
+
+        def get_capitalize_1(type_):
+            return type_.capitalize
+
+        self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True)
+        self.assertEqual(get_capitalize_1(str)('hello'), 'Hello')
+        self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello')
+
+    @requires_specialization
+    def test_property_load_attr_specialization_user_type(self):
+        class G:
+            @property
+            def x(self):
+                return 9
+
+        self._assign_valid_version_or_skip(G)
+
+        def load_x_1(instance):
+            instance.x
+
+        self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True)
+        del load_x_1
+
+        self._no_more_versions(G)
+
+        def load_x_2(instance):
+            instance.x
+
+        self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False)
+
+    @requires_specialization
+    def test_store_attr_specialization_user_type(self):
+        class B:
+            __slots__ = ("bar",)
+
+        self._assign_valid_version_or_skip(B)
+
+        def store_bar_1(type_):
+            type_.bar = 10
+
+        self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True)
+        del store_bar_1
+
+        self._no_more_versions(B)
+
+        def store_bar_2(type_):
+            type_.bar = 10
+
+        self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False)
+
+    @requires_specialization_ft
+    def test_class_call_specialization_user_type(self):
+        class F:
+            def __init__(self):
+                pass
+
+        self._assign_valid_version_or_skip(F)
+
+        def call_class_1(type_):
+            type_()
+
+        self._check_specialization(call_class_1, F, "CALL", should_specialize=True)
+        del call_class_1
+
+        self._no_more_versions(F)
+
+        def call_class_2(type_):
+            type_()
+
+        self._check_specialization(call_class_2, F, "CALL", should_specialize=False)
+
+    @requires_specialization
+    def test_to_bool_specialization_user_type(self):
+        class H:
+            pass
+
+        self._assign_valid_version_or_skip(H)
+
+        def to_bool_1(instance):
+            not instance
+
+        self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True)
+        del to_bool_1
+
+        self._no_more_versions(H)
+
+        def to_bool_2(instance):
+            not instance
+
+        self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_xxlimited.py

@@ -0,0 +1,90 @@
+import unittest
+from test.support import import_helper
+import types
+
+xxlimited = import_helper.import_module('xxlimited')
+xxlimited_35 = import_helper.import_module('xxlimited_35')
+
+
+class CommonTests:
+    module: types.ModuleType
+
+    def test_xxo_new(self):
+        xxo = self.module.Xxo()
+
+    def test_xxo_attributes(self):
+        xxo = self.module.Xxo()
+        with self.assertRaises(AttributeError):
+            xxo.foo
+        with self.assertRaises(AttributeError):
+            del xxo.foo
+
+        xxo.foo = 1234
+        self.assertEqual(xxo.foo, 1234)
+
+        del xxo.foo
+        with self.assertRaises(AttributeError):
+            xxo.foo
+
+    def test_foo(self):
+        # the foo function adds 2 numbers
+        self.assertEqual(self.module.foo(1, 2), 3)
+
+    def test_str(self):
+        self.assertIsSubclass(self.module.Str, str)
+        self.assertIsNot(self.module.Str, str)
+
+        custom_string = self.module.Str("abcd")
+        self.assertEqual(custom_string, "abcd")
+        self.assertEqual(custom_string.upper(), "ABCD")
+
+    def test_new(self):
+        xxo = self.module.new()
+        self.assertEqual(xxo.demo("abc"), "abc")
+
+
+class TestXXLimited(CommonTests, unittest.TestCase):
+    module = xxlimited
+
+    def test_xxo_demo(self):
+        xxo = self.module.Xxo()
+        other = self.module.Xxo()
+        self.assertEqual(xxo.demo("abc"), "abc")
+        self.assertEqual(xxo.demo(xxo), xxo)
+        self.assertEqual(xxo.demo(other), other)
+        self.assertEqual(xxo.demo(0), None)
+
+    def test_error(self):
+        with self.assertRaises(self.module.Error):
+            raise self.module.Error
+
+    def test_buffer(self):
+        xxo = self.module.Xxo()
+        self.assertEqual(xxo.x_exports, 0)
+        b1 = memoryview(xxo)
+        self.assertEqual(xxo.x_exports, 1)
+        b2 = memoryview(xxo)
+        self.assertEqual(xxo.x_exports, 2)
+        b1[0] = 1
+        self.assertEqual(b1[0], 1)
+        self.assertEqual(b2[0], 1)
+
+
+class TestXXLimited35(CommonTests, unittest.TestCase):
+    module = xxlimited_35
+
+    def test_xxo_demo(self):
+        xxo = self.module.Xxo()
+        other = self.module.Xxo()
+        self.assertEqual(xxo.demo("abc"), "abc")
+        self.assertEqual(xxo.demo(0), None)
+
+    def test_roj(self):
+        # the roj function always fails
+        with self.assertRaises(SystemError):
+            self.module.roj(0)
+
+    def test_null(self):
+        null1 = self.module.Null()
+        null2 = self.module.Null()
+        self.assertNotEqual(null1, null2)

Lib/test/test_xxtestfuzz.py

@@ -0,0 +1,25 @@
+import faulthandler
+from test.support import import_helper
+import unittest
+
+_xxtestfuzz = import_helper.import_module('_xxtestfuzz')
+
+
+class TestFuzzer(unittest.TestCase):
+    """To keep our https://github.com/google/oss-fuzz API working."""
+
+    def test_sample_input_smoke_test(self):
+        """This is only a regression test: Check that it doesn't crash."""
+        _xxtestfuzz.run(b"")
+        _xxtestfuzz.run(b"\0")
+        _xxtestfuzz.run(b"{")
+        _xxtestfuzz.run(b" ")
+        _xxtestfuzz.run(b"x")
+        _xxtestfuzz.run(b"1")
+        _xxtestfuzz.run(b"AAAAAAA")
+        _xxtestfuzz.run(b"AAAAAA\0")
+
+
+if __name__ == "__main__":
+    faulthandler.enable()
+    unittest.main()