From 0446bc71a08edb1671a544d7477561f89f2ae7fc Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Fri, 30 Jan 2026 16:51:16 +0900 Subject: [PATCH] Update weakref from v3.14.2 --- Lib/_weakrefset.py | 78 ++------------- Lib/test/test_weakref.py | 71 +++++++------- Lib/weakref.py | 198 ++++++++++----------------------------- 3 files changed, 93 insertions(+), 254 deletions(-) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 489eec714..d1c7fcaee 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ from types import GenericAlias __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -83,21 +43,15 @@ class WeakSet: return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -108,18 +62,12 @@ class WeakSet: return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -136,8 +84,6 @@ class WeakSet: def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -151,8 +97,6 @@ class WeakSet: def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -184,8 +128,6 @@ class WeakSet: def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index f421d745a..bfecca439 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -13,7 +13,7 @@ import random import textwrap from test import support -from test.support import script_helper, ALWAYS_EQ, suppress_immortalization +from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect from test.support import import_helper from test.support import threading_helper @@ -289,7 +289,7 @@ class ReferencesTestCase(TestBase): self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_reuse(self): o = C() proxy1 = weakref.proxy(o) @@ -337,7 +337,7 @@ class ReferencesTestCase(TestBase): self.assertIn("__bytes__", dir(weakref.proxy(instance))) self.assertEqual(bytes(weakref.proxy(instance)), b"bytes") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_index(self): class C: def __index__(self): @@ -346,7 +346,7 @@ class ReferencesTestCase(TestBase): p = weakref.proxy(o) self.assertEqual(operator.index(p), 10) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_div(self): class C: def __floordiv__(self, other): @@ -359,7 +359,7 @@ class ReferencesTestCase(TestBase): p //= 5 self.assertEqual(p, 21) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_matmul(self): class C: def __matmul__(self, other): @@ -383,11 +383,11 @@ class ReferencesTestCase(TestBase): # was not honored, and was broken in different ways for # PyWeakref_NewRef() and PyWeakref_NewProxy(). (Two tests.) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_ref_without_callback(self): self.check_shared_without_callback(weakref.ref) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_proxy_without_callback(self): self.check_shared_without_callback(weakref.proxy) @@ -409,7 +409,7 @@ class ReferencesTestCase(TestBase): p2 = makeref(o) self.assertIs(p1, p2, "callbacks were None, NULL in the C API") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callable_proxy(self): o = Callable() ref1 = weakref.proxy(o) @@ -439,7 +439,7 @@ class ReferencesTestCase(TestBase): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -449,7 +449,7 @@ class ReferencesTestCase(TestBase): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -504,7 +504,7 @@ class ReferencesTestCase(TestBase): # Calls proxy.__next__ self.assertEqual(list(weak_it), [4, 5, 6]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_bad_next(self): # bpo-44720: PyIter_Next() shouldn't be called if the reference # isn't an iterator. @@ -594,7 +594,7 @@ class ReferencesTestCase(TestBase): self.assertEqual(weakref.getweakrefs(1), [], "list of refs does not match for int") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newstyle_number_ops(self): class F(float): pass @@ -668,7 +668,6 @@ class ReferencesTestCase(TestBase): # deallocation of c2. del c2 - @suppress_immortalization() def test_callback_in_cycle(self): import gc @@ -761,8 +760,7 @@ class ReferencesTestCase(TestBase): del c1, c2, C, D gc.collect() - @unittest.expectedFailure # TODO: RUSTPYTHON - @suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_in_cycle_resurrection(self): import gc @@ -809,7 +807,7 @@ class ReferencesTestCase(TestBase): gc.collect() self.assertEqual(alist, []) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callbacks_on_callback(self): import gc @@ -848,11 +846,11 @@ class ReferencesTestCase(TestBase): gc.collect() self.assertEqual(alist, []) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_gc_during_ref_creation(self): self.check_gc_during_creation(weakref.ref) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_gc_during_proxy_creation(self): self.check_gc_during_creation(weakref.proxy) @@ -901,7 +899,7 @@ class ReferencesTestCase(TestBase): # No exception should be raised here gc.collect() - @suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .A'> != None def test_classes(self): # Check that classes are weakrefable. class A(object): @@ -1003,7 +1001,7 @@ class ReferencesTestCase(TestBase): del root gc.collect() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute(self): x = Object(1) callback = lambda ref: None @@ -1013,7 +1011,7 @@ class ReferencesTestCase(TestBase): ref2 = weakref.ref(x) self.assertIsNone(ref2.__callback__) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute_after_deletion(self): x = Object(1) ref = weakref.ref(x, self.callback) @@ -1065,7 +1063,6 @@ class ReferencesTestCase(TestBase): class SubclassableWeakrefTestCase(TestBase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_refs(self): class MyRef(weakref.ref): def __init__(self, ob, callback=None, value=42): @@ -1084,7 +1081,7 @@ class SubclassableWeakrefTestCase(TestBase): self.assertIsNone(mr()) self.assertTrue(mr.called) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_refs_dont_replace_standard_refs(self): class MyRef(weakref.ref): pass @@ -1128,7 +1125,7 @@ class SubclassableWeakrefTestCase(TestBase): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" @@ -1336,11 +1333,11 @@ class MappingTestCase(TestBase): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_keyed_len_cycles(self): self.check_len_cycles(weakref.WeakKeyDictionary, lambda k: (k, 1)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_valued_len_cycles(self): self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1368,11 +1365,11 @@ class MappingTestCase(TestBase): self.assertGreaterEqual(n2, 0) self.assertLessEqual(n2, n1) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_keyed_len_race(self): self.check_len_race(weakref.WeakKeyDictionary, lambda k: (k, 1)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_valued_len_race(self): self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1873,7 +1870,7 @@ class MappingTestCase(TestBase): self.assertEqual(len(d), 1) self.assertEqual(list(d.items()), [('something else', o2)]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_keyed_bad_delitem(self): d = weakref.WeakKeyDictionary() o = Object('1') @@ -2054,12 +2051,13 @@ class MappingTestCase(TestBase): raise exc[0] @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) - @unittest.skip('TODO: RUSTPYTHON; occasionally crash (malloc corruption)') + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threaded_weak_key_dict_deepcopy(self): @@ -2067,14 +2065,15 @@ class MappingTestCase(TestBase): # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) - @unittest.skip('TODO: RUSTPYTHON; occasionally crash (malloc corruption)') + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) - @unittest.skip('TODO: RUSTPYTHON; occasionally crash (malloc corruption)') + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threaded_weak_value_dict_deepcopy(self): @@ -2259,7 +2258,7 @@ class FinalizeTestCase(unittest.TestCase): assert f3.atexit == True assert f4.atexit == True - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Windows') + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; Windows") def test_atexit(self): prog = ('from test.test_weakref import FinalizeTestCase;'+ 'FinalizeTestCase.run_in_child()') @@ -2270,7 +2269,7 @@ class FinalizeTestCase(unittest.TestCase): class ModuleTestCase(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_names(self): for name in ('ReferenceType', 'ProxyType', 'CallableProxyType', 'WeakMethod', 'WeakSet', 'WeakKeyDictionary', 'WeakValueDictionary'): @@ -2370,9 +2369,7 @@ OK __test__ = {'libreftest' : libreftest} def load_tests(loader, tests, pattern): - # TODO: RUSTPYTHON - # The doctest fails - # tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/weakref.py b/Lib/weakref.py index 25b70927e..94e427814 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -19,7 +19,7 @@ from _weakref import ( ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -105,34 +105,14 @@ class WeakValueDictionary(_collections_abc.MutableMapping): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -140,18 +120,12 @@ class WeakValueDictionary(_collections_abc.MutableMapping): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -162,38 +136,28 @@ class WeakValueDictionary(_collections_abc.MutableMapping): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -207,21 +171,15 @@ class WeakValueDictionary(_collections_abc.MutableMapping): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -235,23 +193,15 @@ class WeakValueDictionary(_collections_abc.MutableMapping): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -259,8 +209,6 @@ class WeakValueDictionary(_collections_abc.MutableMapping): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -279,16 +227,12 @@ class WeakValueDictionary(_collections_abc.MutableMapping): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -308,9 +252,7 @@ class WeakValueDictionary(_collections_abc.MutableMapping): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -369,57 +311,22 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -429,11 +336,10 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -441,11 +347,10 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -459,26 +364,23 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -493,7 +395,6 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -501,7 +402,6 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None):