Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let trackref release classes defined at run time (#7922)

Adrianafbd2c9320aa
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 0bbe68def..2882b7f96 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -14,7 +14,6 @@ This library has a minimal performance impact.
from __future__ import annotations
-from collections import defaultdict
from operator import itemgetter
from time import monotonic_ns
from types import NoneType
@@ -28,8 +27,8 @@ if TYPE_CHECKING:
from typing_extensions import Self
-live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
- WeakKeyDictionary
+live_refs: WeakKeyDictionary[type, WeakKeyDictionary[object, float]] = (
+ WeakKeyDictionary()
)
@@ -41,7 +40,11 @@ class object_ref:
def __new__(cls, *args: Any, **kwargs: Any) -> Self:
obj = object.__new__(cls)
- live_refs[cls][obj] = monotonic_ns()
+ try:
+ refs = live_refs[cls]
+ except KeyError:
+ refs = live_refs[cls] = WeakKeyDictionary()
+ refs[obj] = monotonic_ns()
return obj
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 5458aa603..9585c6d83 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -124,3 +124,13 @@ def test_iter_all():
o2 = Bar() # noqa: F841
o3 = Foo()
assert set(trackref.iter_all("Foo")) == {o1, o3}
+
+
+def test_run_time_classes() -> None:
+ for _ in range(10):
+ base = type("Baz", (trackref.object_ref,), {})
+ base()
+ del base
+ garbage_collect()
+ assert not list(trackref.iter_all("Baz"))
+ assert sum(1 for cls in trackref.live_refs if cls.__name__ == "Baz") == 0