Align bytes and bytearray operations with Python 3.15 (#9165)

Correct quote escaping, replacement count keyword binding, signed-minimum hex grouping, and clear storage/export handling. Migrate canonical bytes coverage through fail-closed preservation of the existing RustPython customizations, and retire only two freshly verified formatting guards.

Assisted-by: OpenAI Codex:model version unavailable

scripts/update_lib/cmd_migrate.py

@@ -22,11 +22,13 @@ import argparse
 import ast
 import builtins
 import collections
+import copy
 import hashlib
 import json
 import pathlib
 import shutil
 import sys
+import textwrap
 
 sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
 
@@ -49,6 +51,144 @@ else:
 EXCEPTION_LOOP = """for excname in PYTHON2_EXCEPTIONS:
     NAME_MAPPING[("exceptions", excname)] = ("builtins", excname)
 """
+BYTES_CAPI_UPSTREAM = (
+    "    _testlimitedcapi = import_helper.import_module('_testlimitedcapi')\n"
+)
+BYTES_CAPI_LOCAL = """    # TODO: RUSTPYTHON; import_helper.import_module here cause the entire test stopping
+    # _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
+    try:
+        import _testlimitedcapi
+    except ImportError:
+        _testlimitedcapi = None
+"""
+BYTES_ALLOCATION_COMMENTS = {
+    "test_alloc": """    # NOTE: RUSTPYTHON:
+    #
+    # The second instance of self.assertGreater was replaced with
+    # self.assertGreaterEqual since, in RustPython, the underlying storage
+    # is a Vec which doesn't require trailing null byte.
+""",
+    "test_init_alloc": """    # NOTE: RUSTPYTHON:
+    #
+    # The usages of self.assertGreater were replaced with
+    # self.assertGreaterEqual since, in RustPython, the underlying storage
+    # is a Vec which doesn't require trailing null byte.
+""",
+}
+
+
+def _bytes_bindings(root, name):
+    """Find bindings conservatively, including nested or indirect shadows."""
+    for node in ast.walk(root):
+        if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
+            bound = node.id
+        elif isinstance(node, ast.alias):
+            bound = node.asname or node.name.split(".")[0]
+        elif isinstance(
+            node,
+            (
+                ast.FunctionDef,
+                ast.AsyncFunctionDef,
+                ast.ClassDef,
+                ast.ExceptHandler,
+                ast.MatchAs,
+                ast.MatchStar,
+            ),
+        ):
+            bound = node.name
+        elif isinstance(node, ast.MatchMapping):
+            bound = node.rest
+        elif isinstance(node, ast.arg):
+            bound = node.arg
+        elif isinstance(node, (ast.Global, ast.Nonlocal)):
+            bound = name if name in node.names else None
+        else:
+            continue
+        if bound == name:
+            yield node
+
+
+def _bytes_owner(contents):
+    tree = ast.parse(contents)
+    owner = _unique_anchor(
+        contents,
+        tree,
+        lambda node: isinstance(node, ast.ClassDef) and node.name == "ByteArrayTest",
+        direct=True,
+    )
+    if list(_bytes_bindings(tree, "ByteArrayTest")) != [owner]:
+        raise ValueError("Shadowed bytes test class")
+    return owner
+
+
+def _verify_bytes_capi(contents: str, block: str) -> ast.stmt:
+    owner = _bytes_owner(contents)
+    expected = ast.parse(textwrap.dedent(block)).body[0]
+    anchor = _unique_anchor(
+        contents, owner, lambda node: ast.dump(node) == ast.dump(expected), direct=True
+    )
+    allowed = set(ast.walk(anchor))
+    if any(node not in allowed for node in _bytes_bindings(owner, "_testlimitedcapi")):
+        raise ValueError("Shadowed optional bytes C API import")
+    start = anchor.lineno - 1 - (2 if block == BYTES_CAPI_LOCAL else 0)
+    anchored_text = "".join(
+        contents.splitlines(keepends=True)[start : anchor.end_lineno]
+    )
+    if start < 0 or anchored_text != block:
+        raise ValueError("Missing, changed, or ambiguous optional bytes C API import")
+    return anchor
+
+
+def _preserve_bytes_allocations(old: str, new: str) -> str:
+    """Keep only the existing three Vec allocation assertion adaptations."""
+    owners = [_bytes_owner(contents) for contents in (old, new)]
+    replacements = []
+    for name, count in (("test_alloc", 1), ("test_init_alloc", 2)):
+        methods = [
+            _unique_anchor(
+                contents,
+                owner,
+                lambda node: (
+                    isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+                    and node.name == name
+                ),
+                direct=True,
+            )
+            for contents, owner in zip((old, new), owners)
+        ]
+        if any(
+            list(_bytes_bindings(owner, name)) != [method]
+            for owner, method in zip(owners, methods)
+        ):
+            raise ValueError("Shadowed bytes allocation test")
+        if any(not isinstance(method, ast.FunctionDef) for method in methods):
+            raise ValueError("Changed bytes allocation test kind")
+        normalized = copy.deepcopy(methods[0])
+        expected = ast.parse("self.assertGreaterEqual(alloc, len(b))", mode="eval").body
+        matches = [
+            node
+            for node in ast.walk(normalized)
+            if isinstance(node, ast.Call) and ast.dump(node) == ast.dump(expected)
+        ]
+        if len(matches) != count:
+            raise ValueError("Changed local bytes allocation contract")
+        for node in matches:
+            node.func.attr = "assertGreater"
+        if ast.dump(normalized) != ast.dump(methods[1]):
+            raise ValueError("Changed canonical bytes allocation contract")
+        old_lines = old.splitlines(keepends=True)
+        comment = BYTES_ALLOCATION_COMMENTS[name]
+        start = methods[0].lineno - 1 - len(comment.splitlines())
+        if start < 0 or "".join(old_lines[start : methods[0].lineno - 1]) != comment:
+            raise ValueError("Changed local bytes allocation explanation")
+        local = "".join(old_lines[start : methods[0].end_lineno])
+        replacements.append((methods[1].lineno - 1, methods[1].end_lineno, local))
+    lines = new.splitlines(keepends=True)
+    for start, end, local in sorted(replacements, reverse=True):
+        lines[start:end] = [local]
+    result = "".join(lines)
+    ast.parse(result)
+    return result
 
 
 def _unique_anchor(contents, root, predicate, *, direct=False, exact=None):
@@ -153,6 +293,12 @@ def preserve_migration_customizations(path: pathlib.Path, old: str, new: str) ->
         _verify_call(old, LOCAL_CALL)
         _verify_call(new, CANONICAL_CALL)
         return new.replace(CANONICAL_CALL, LOCAL_CALL, 1)
+    if parts == ("Lib", "test", "test_bytes.py"):
+        _verify_bytes_capi(old, BYTES_CAPI_LOCAL)
+        anchor = _verify_bytes_capi(new, BYTES_CAPI_UPSTREAM)
+        lines = new.splitlines(keepends=True)
+        lines[anchor.lineno - 1 : anchor.end_lineno] = [BYTES_CAPI_LOCAL]
+        return _preserve_bytes_allocations(old, "".join(lines))
     return new
 
 

scripts/update_lib/tests/test_migrate.py

@@ -11,6 +11,9 @@ from pathlib import Path
 from unittest.mock import patch
 
 from update_lib.cmd_migrate import (
+    BYTES_ALLOCATION_COMMENTS,
+    BYTES_CAPI_LOCAL,
+    BYTES_CAPI_UPSTREAM,
     CANONICAL_CALL,
     EXCEPTION_LOOP,
     JIT_ERROR,
@@ -37,6 +40,240 @@ from update_lib.patch_spec import (
 )
 
 
+class TestBytesPreservation(unittest.TestCase):
+    path = Path("Lib/test/test_bytes.py")
+    prefix = "class ByteArrayTest(BaseBytesTest, unittest.TestCase):\n"
+    allocations = """
+    def test_alloc(self):
+        b = bytearray()
+        alloc = b.__alloc__()
+        self.assertGreaterEqual(alloc, 0)
+        # existing allocation adaptation
+        self.assertGreaterEqual(alloc, len(b))
+
+    def test_init_alloc(self):
+        b = bytearray()
+        def g():
+            alloc = b.__alloc__()
+            self.assertGreaterEqual(alloc, len(b))
+            yield 1
+        b.__init__(g())
+        alloc = b.__alloc__()
+        self.assertGreaterEqual(alloc, len(b))
+"""
+    canonical_allocations = allocations.replace(
+        "assertGreaterEqual(alloc, len(b))", "assertGreater(alloc, len(b))"
+    ).replace("        # existing allocation adaptation\n", "")
+    for name, comment in BYTES_ALLOCATION_COMMENTS.items():
+        allocations = allocations.replace(
+            f"    def {name}(", comment + f"    def {name}("
+        )
+    old = prefix + BYTES_CAPI_LOCAL + allocations
+    canonical = prefix + BYTES_CAPI_UPSTREAM + canonical_allocations
+
+    def assert_rejected(self, changed, *, local):
+        with self.assertRaises(ValueError):
+            preserve_migration_customizations(
+                self.path,
+                changed if local else self.old,
+                self.canonical if local else changed,
+            )
+
+    def test_preserves_existing_blocks_and_is_idempotent(self):
+        source = self.canonical + "\nnew_canonical_content = True\n"
+        expected = self.old + "\nnew_canonical_content = True\n"
+        result = preserve_migration_customizations(self.path, self.old, source)
+        self.assertEqual(result, expected)
+        self.assertEqual(
+            preserve_migration_customizations(self.path, result, source), result
+        )
+
+    def test_earlier_nested_import_and_decoy_text_are_untouched(self):
+        earlier = (
+            "class BaseBytesTest:\n    def test_sq_item(self):\n    "
+            + BYTES_CAPI_UPSTREAM
+        )
+        decoy = 'description = """\n' + BYTES_CAPI_UPSTREAM + '"""\n'
+        prefix = earlier + decoy
+        self.assertEqual(
+            preserve_migration_customizations(
+                self.path, prefix + self.old, prefix + self.canonical
+            ),
+            prefix + self.old,
+        )
+
+    def test_import_missing_duplicate_changed_and_moved_fail_closed(self):
+        for local, block, allocations in (
+            (True, BYTES_CAPI_LOCAL, self.allocations),
+            (False, BYTES_CAPI_UPSTREAM, self.canonical_allocations),
+        ):
+            source = self.prefix + block + allocations
+            for changed in (
+                self.prefix + "    pass\n" + allocations,
+                source + block,
+                source + source,
+                source.replace("ByteArrayTest", "OtherTest"),
+                source.replace("_testlimitedcapi", "_other"),
+                source.replace("ImportError", "Exception")
+                if local
+                else source.replace("import_module", "other"),
+                "if True:\n"
+                + "".join("    " + line for line in source.splitlines(True)),
+            ):
+                with self.subTest(local=local, changed=changed):
+                    self.assert_rejected(changed, local=local)
+
+    def test_import_and_owner_shadows_fail_closed(self):
+        shadows = (
+            "    _testlimitedcapi = None\n",
+            "    del _testlimitedcapi\n",
+            "    import other as _testlimitedcapi\n",
+            "    def _testlimitedcapi(self): pass\n",
+            "    async def _testlimitedcapi(self): pass\n",
+            "    class _testlimitedcapi: pass\n",
+            "    try:\n        pass\n"
+            "    except Exception as _testlimitedcapi:\n        pass\n",
+            "    match None:\n        case _testlimitedcapi: pass\n",
+            "    global _testlimitedcapi\n",
+            "ByteArrayTest = Other\n",
+            "del ByteArrayTest\n",
+            "from other import ByteArrayTest\n",
+            "def ByteArrayTest(): pass\n",
+            "async def ByteArrayTest(): pass\n",
+            "try:\n    pass\nexcept Exception as ByteArrayTest:\n    pass\n",
+            "match None:\n    case ByteArrayTest: pass\n",
+        )
+        for local, source in ((True, self.old), (False, self.canonical)):
+            for shadow in shadows:
+                with self.subTest(local=local, shadow=shadow):
+                    self.assert_rejected(source + shadow, local=local)
+
+    def test_decoy_cannot_validate_changed_real_import(self):
+        for local, block, allocations in (
+            (True, BYTES_CAPI_LOCAL, self.allocations),
+            (False, BYTES_CAPI_UPSTREAM, self.canonical_allocations),
+        ):
+            decoy = '    """\n' + block + '    """\n'
+            changed = (
+                self.prefix
+                + decoy
+                + block.replace("_testlimitedcapi =", "_testlimitedcapi  =")
+                + allocations
+            )
+            with self.subTest(local=local):
+                self.assert_rejected(changed, local=local)
+        self.assert_rejected(
+            self.canonical.replace(
+                BYTES_CAPI_UPSTREAM,
+                BYTES_CAPI_UPSTREAM.rstrip() + "; unrelated = 1\n",
+            ),
+            local=False,
+        )
+        self.assert_rejected(
+            self.old.replace("# TODO: RUSTPYTHON;", "# changed existing comment;"),
+            local=True,
+        )
+
+    def test_changed_allocation_methods_fail_closed(self):
+        for local, source in ((True, self.old), (False, self.canonical)):
+            for changed in (
+                source.replace("len(b)", "len(other)", 1),
+                source.replace(
+                    "def test_alloc(self):",
+                    "def test_alloc(self):\n        changed = True",
+                ),
+                source.replace("def test_alloc(self):", "def renamed(self):"),
+                source.replace("def test_alloc(self):", "async def test_alloc(self):"),
+                source + "\n    def test_alloc(self): pass\n",
+            ):
+                with self.subTest(local=local, changed=changed):
+                    self.assert_rejected(changed, local=local)
+        for changed in (
+            self.old.replace(
+                "assertGreaterEqual(alloc, len(b))", "assertEqual(alloc, len(b))", 1
+            ),
+            self.old.replace("self.assertGreaterEqual(alloc, len(b))", "pass", 1),
+            self.old.replace(
+                "self.assertGreaterEqual(alloc, len(b))",
+                "self.assertGreaterEqual(alloc, len(b)); "
+                "self.assertGreaterEqual(alloc, len(b))",
+                1,
+            ),
+        ):
+            self.assert_rejected(changed, local=True)
+
+    def test_allocation_method_shadows_fail_closed(self):
+        for name in ("test_alloc", "test_init_alloc"):
+            for local, source in ((True, self.old), (False, self.canonical)):
+                for shadow in (
+                    f"    {name} = None\n",
+                    f"    del {name}\n",
+                    f"    from other import {name}\n",
+                    f"    async def {name}(self): pass\n",
+                    f"    class {name}: pass\n",
+                    "    try:\n        pass\n"
+                    f"    except Exception as {name}:\n        pass\n",
+                    f"    match None:\n        case {name}: pass\n",
+                ):
+                    with self.subTest(name=name, local=local, shadow=shadow):
+                        self.assert_rejected(source + shadow, local=local)
+
+    def test_changed_allocation_explanation_fails_closed(self):
+        for comment in BYTES_ALLOCATION_COMMENTS.values():
+            for replacement in ("", comment.replace("RUSTPYTHON", "changed")):
+                with self.subTest(comment=comment, replacement=replacement):
+                    self.assert_rejected(
+                        self.old.replace(comment, replacement), local=True
+                    )
+
+    def test_other_paths_are_untouched(self):
+        for path in ("Lib/test/other.py", "Lib/other/test_bytes.py"):
+            self.assertEqual(
+                preserve_migration_customizations(Path(path), "old", "new"), "new"
+            )
+
+    def test_file_migration_preserves_guards_and_rejection_writes_nothing(self):
+        guard = (
+            "\n    @unittest.expectedFailure  # TODO: RUSTPYTHON\n"
+            "    def test_example(self):\n        pass\n"
+        )
+        canonical_test = "\n    def test_example(self):\n        pass\n"
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            output = root / self.path
+            output.parent.mkdir(parents=True)
+            output.write_text("import unittest\n" + self.old + guard, encoding="utf-8")
+            source = root / "canonical.py"
+            source.write_text(
+                "import unittest\n" + self.canonical + canonical_test, encoding="utf-8"
+            )
+            before_source = source.read_bytes()
+            before_guards = patches_to_json(extract_patches(output.read_text()))
+            module_file = root / "scripts/update_lib/cmd_migrate.py"
+            with patch("update_lib.cmd_migrate.__file__", str(module_file)):
+                patch_file(source, output, verbose=False)
+                self.assertEqual(
+                    output.read_text(), "import unittest\n" + self.old + guard
+                )
+                self.assertEqual(source.read_bytes(), before_source)
+                self.assertEqual(
+                    patches_to_json(extract_patches(output.read_text())), before_guards
+                )
+                self.assertEqual(
+                    patch_single_content(source, output), output.read_text()
+                )
+                before_output = output.read_bytes()
+                source.write_text(
+                    source.read_text().replace("ByteArrayTest", "OtherTest"),
+                    encoding="utf-8",
+                )
+                rejected_source = source.read_bytes()
+                with self.assertRaises(ValueError):
+                    patch_file(source, output, verbose=False)
+                self.assertEqual(output.read_bytes(), before_output)
+                self.assertEqual(source.read_bytes(), rejected_source)
+
+
 class TestSupportAssignments(unittest.TestCase):
     names = ["_available", "requires_available", "requires_unavailable"]
     exports = names[1:]