Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts

Kshitij Sharma49337bd2ae09
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7a393925e..c8f921ff3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
the start position of the match, and the ending (regarding the entire text).
"""
+
def _chunk_iter():
offset = len(text)
while True:
@@ -158,6 +159,7 @@ def memoizemethod_noargs(method):
if self not in cache:
cache[self] = method(self, *args, **kwargs)
return cache[self]
+
return new_method
@@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes):
return True
+@deprecated
+class WeakKeyCache:
+
+ def __init__(self, default_factory):
+ self.default_factory = default_factory
+ self._weakdict = weakref.WeakKeyDictionary()
+
+ def __getitem__(self, key):
+ if key not in self._weakdict:
+ self._weakdict[key] = self.default_factory(key)
+ return self._weakdict[key]
+
+
@deprecated
def retry_on_eintr(function, *args, **kw):
"""Run a function and retry it while getting EINTR errors"""