2008-02-05 08:20:01 +08:00
|
|
|
# Access WeakSet through the weakref module.
|
|
|
|
# This code is separated-out because it is needed
|
|
|
|
# by abc.py to load everything else at startup.
|
|
|
|
|
|
|
|
from _weakref import ref
|
2020-04-14 12:54:40 +08:00
|
|
|
from types import GenericAlias
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
__all__ = ['WeakSet']
|
|
|
|
|
2010-01-09 01:54:23 +08:00
|
|
|
|
|
|
|
class _IterationGuard:
|
|
|
|
# This context manager registers itself in the current iterators of the
|
|
|
|
# weak container, such as to delay all removals until the context manager
|
|
|
|
# exits.
|
|
|
|
# This technique should be relatively thread-safe (since sets are).
|
|
|
|
|
|
|
|
def __init__(self, weakcontainer):
|
|
|
|
# Don't create cycles
|
|
|
|
self.weakcontainer = ref(weakcontainer)
|
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
w = self.weakcontainer()
|
|
|
|
if w is not None:
|
|
|
|
w._iterating.add(self)
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __exit__(self, e, t, b):
|
|
|
|
w = self.weakcontainer()
|
|
|
|
if w is not None:
|
|
|
|
s = w._iterating
|
|
|
|
s.remove(self)
|
|
|
|
if not s:
|
|
|
|
w._commit_removals()
|
|
|
|
|
|
|
|
|
2008-02-05 08:20:01 +08:00
|
|
|
class WeakSet:
|
|
|
|
def __init__(self, data=None):
|
|
|
|
self.data = set()
|
|
|
|
def _remove(item, selfref=ref(self)):
|
|
|
|
self = selfref()
|
|
|
|
if self is not None:
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._iterating:
|
|
|
|
self._pending_removals.append(item)
|
|
|
|
else:
|
|
|
|
self.data.discard(item)
|
2008-02-05 08:20:01 +08:00
|
|
|
self._remove = _remove
|
2010-01-09 01:54:23 +08:00
|
|
|
# A list of keys to be removed
|
|
|
|
self._pending_removals = []
|
|
|
|
self._iterating = set()
|
2008-02-05 08:20:01 +08:00
|
|
|
if data is not None:
|
|
|
|
self.update(data)
|
|
|
|
|
2010-01-09 01:54:23 +08:00
|
|
|
def _commit_removals(self):
|
bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921)
Fixes:
Traceback (most recent call last):
File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in <module>
sys.exit(main())
File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main
test_all_tasks_threading()
File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading
results.append(f.result())
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result
return self.__get_result()
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run
_cancel_all_tasks(loop)
File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks
to_cancel = tasks.all_tasks(loop)
File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks
tasks = list(_all_tasks)
File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__
with _IterationGuard(self):
File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__
w._commit_removals()
File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals
discard(l.pop())
IndexError: pop from empty list
Also fixes:
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d8180; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d81a0; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x000056548f1e24a0; dead>
See: https://github.com/agronholm/anyio/issues/362#issuecomment-904424310
See also: https://bugs.python.org/issue29519
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
2021-08-29 01:07:37 +08:00
|
|
|
pop = self._pending_removals.pop
|
2010-01-09 01:54:23 +08:00
|
|
|
discard = self.data.discard
|
bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921)
Fixes:
Traceback (most recent call last):
File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in <module>
sys.exit(main())
File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main
test_all_tasks_threading()
File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading
results.append(f.result())
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result
return self.__get_result()
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run
_cancel_all_tasks(loop)
File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks
to_cancel = tasks.all_tasks(loop)
File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks
tasks = list(_all_tasks)
File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__
with _IterationGuard(self):
File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__
w._commit_removals()
File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals
discard(l.pop())
IndexError: pop from empty list
Also fixes:
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d8180; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d81a0; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x000056548f1e24a0; dead>
See: https://github.com/agronholm/anyio/issues/362#issuecomment-904424310
See also: https://bugs.python.org/issue29519
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
2021-08-29 01:07:37 +08:00
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
item = pop()
|
|
|
|
except IndexError:
|
|
|
|
return
|
|
|
|
discard(item)
|
2010-01-09 01:54:23 +08:00
|
|
|
|
2008-02-05 08:20:01 +08:00
|
|
|
def __iter__(self):
|
2010-01-09 01:54:23 +08:00
|
|
|
with _IterationGuard(self):
|
|
|
|
for itemref in self.data:
|
|
|
|
item = itemref()
|
|
|
|
if item is not None:
|
2013-12-18 07:28:36 +08:00
|
|
|
# Caveat: the iterator will keep a strong reference to
|
|
|
|
# `item` until it is resumed or closed.
|
2010-01-09 01:54:23 +08:00
|
|
|
yield item
|
2008-02-05 08:20:01 +08:00
|
|
|
|
2008-05-19 00:27:29 +08:00
|
|
|
def __len__(self):
|
2012-03-01 23:26:35 +08:00
|
|
|
return len(self.data) - len(self._pending_removals)
|
2008-05-19 00:27:29 +08:00
|
|
|
|
2008-02-05 08:20:01 +08:00
|
|
|
def __contains__(self, item):
|
2010-12-03 15:55:44 +08:00
|
|
|
try:
|
|
|
|
wr = ref(item)
|
|
|
|
except TypeError:
|
|
|
|
return False
|
|
|
|
return wr in self.data
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def __reduce__(self):
|
2022-04-07 01:00:14 +08:00
|
|
|
return self.__class__, (list(self),), self.__getstate__()
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def add(self, item):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-02-05 08:20:01 +08:00
|
|
|
self.data.add(ref(item, self._remove))
|
|
|
|
|
|
|
|
def clear(self):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-02-05 08:20:01 +08:00
|
|
|
self.data.clear()
|
|
|
|
|
|
|
|
def copy(self):
|
|
|
|
return self.__class__(self)
|
|
|
|
|
|
|
|
def pop(self):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-02-05 08:20:01 +08:00
|
|
|
while True:
|
2008-05-18 15:46:13 +08:00
|
|
|
try:
|
|
|
|
itemref = self.data.pop()
|
|
|
|
except KeyError:
|
2017-04-05 14:37:24 +08:00
|
|
|
raise KeyError('pop from empty WeakSet') from None
|
2008-02-05 08:20:01 +08:00
|
|
|
item = itemref()
|
|
|
|
if item is not None:
|
|
|
|
return item
|
|
|
|
|
|
|
|
def remove(self, item):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-02-05 08:20:01 +08:00
|
|
|
self.data.remove(ref(item))
|
|
|
|
|
|
|
|
def discard(self, item):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-02-05 08:20:01 +08:00
|
|
|
self.data.discard(ref(item))
|
|
|
|
|
|
|
|
def update(self, other):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2012-03-05 03:20:34 +08:00
|
|
|
for element in other:
|
|
|
|
self.add(element)
|
2010-01-09 01:54:23 +08:00
|
|
|
|
2008-05-19 00:27:29 +08:00
|
|
|
def __ior__(self, other):
|
|
|
|
self.update(other)
|
|
|
|
return self
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def difference(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
newset = self.copy()
|
|
|
|
newset.difference_update(other)
|
|
|
|
return newset
|
2008-02-05 08:20:01 +08:00
|
|
|
__sub__ = difference
|
|
|
|
|
|
|
|
def difference_update(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
self.__isub__(other)
|
2008-05-19 00:27:29 +08:00
|
|
|
def __isub__(self, other):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-05-19 00:27:29 +08:00
|
|
|
if self is other:
|
|
|
|
self.data.clear()
|
|
|
|
else:
|
|
|
|
self.data.difference_update(ref(item) for item in other)
|
|
|
|
return self
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def intersection(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
return self.__class__(item for item in other if item in self)
|
2008-02-05 08:20:01 +08:00
|
|
|
__and__ = intersection
|
|
|
|
|
|
|
|
def intersection_update(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
self.__iand__(other)
|
2008-05-19 00:27:29 +08:00
|
|
|
def __iand__(self, other):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-05-19 00:27:29 +08:00
|
|
|
self.data.intersection_update(ref(item) for item in other)
|
|
|
|
return self
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def issubset(self, other):
|
|
|
|
return self.data.issubset(ref(item) for item in other)
|
2012-03-05 12:15:38 +08:00
|
|
|
__le__ = issubset
|
2008-02-05 08:20:01 +08:00
|
|
|
|
2012-03-05 12:15:38 +08:00
|
|
|
def __lt__(self, other):
|
2017-05-18 22:35:54 +08:00
|
|
|
return self.data < set(map(ref, other))
|
2008-05-19 00:27:29 +08:00
|
|
|
|
2008-02-05 08:20:01 +08:00
|
|
|
def issuperset(self, other):
|
|
|
|
return self.data.issuperset(ref(item) for item in other)
|
2012-03-05 12:15:38 +08:00
|
|
|
__ge__ = issuperset
|
2008-02-05 08:20:01 +08:00
|
|
|
|
2012-03-05 12:15:38 +08:00
|
|
|
def __gt__(self, other):
|
2017-05-18 22:35:54 +08:00
|
|
|
return self.data > set(map(ref, other))
|
2008-05-19 00:27:29 +08:00
|
|
|
|
|
|
|
def __eq__(self, other):
|
2009-05-18 01:32:20 +08:00
|
|
|
if not isinstance(other, self.__class__):
|
|
|
|
return NotImplemented
|
2017-05-18 22:35:54 +08:00
|
|
|
return self.data == set(map(ref, other))
|
2008-05-19 00:27:29 +08:00
|
|
|
|
2008-02-05 08:20:01 +08:00
|
|
|
def symmetric_difference(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
newset = self.copy()
|
|
|
|
newset.symmetric_difference_update(other)
|
|
|
|
return newset
|
2008-02-05 08:20:01 +08:00
|
|
|
__xor__ = symmetric_difference
|
|
|
|
|
|
|
|
def symmetric_difference_update(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
self.__ixor__(other)
|
2008-05-19 00:27:29 +08:00
|
|
|
def __ixor__(self, other):
|
2010-01-09 01:54:23 +08:00
|
|
|
if self._pending_removals:
|
|
|
|
self._commit_removals()
|
2008-05-19 00:27:29 +08:00
|
|
|
if self is other:
|
|
|
|
self.data.clear()
|
|
|
|
else:
|
2012-03-05 03:47:05 +08:00
|
|
|
self.data.symmetric_difference_update(ref(item, self._remove) for item in other)
|
2008-05-19 00:27:29 +08:00
|
|
|
return self
|
2008-02-05 08:20:01 +08:00
|
|
|
|
|
|
|
def union(self, other):
|
2012-03-05 03:47:05 +08:00
|
|
|
return self.__class__(e for s in (self, other) for e in s)
|
2008-02-05 08:20:01 +08:00
|
|
|
__or__ = union
|
2008-05-19 00:27:29 +08:00
|
|
|
|
|
|
|
def isdisjoint(self, other):
|
|
|
|
return len(self.intersection(other)) == 0
|
2019-05-21 01:01:07 +08:00
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return repr(self.data)
|
2020-04-14 12:54:40 +08:00
|
|
|
|
|
|
|
__class_getitem__ = classmethod(GenericAlias)
|