Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix LocalCache evicting a different key when updating an existing key (#8113)

Anish Patela4109754c56b
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 47d1de096..5598fe9e4 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -94,8 +94,9 @@ class LocalCache(OrderedDict[_KT, _VT]):
if self.limit is not None:
if self.limit == 0:
return
- while len(self) >= self.limit:
- self.popitem(last=False)
+ if key not in self:
+ while len(self) >= self.limit:
+ self.popitem(last=False)
super().__setitem__(key, value)
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 555a0bd50..55bef53b2 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -324,6 +324,18 @@ class TestLocalCache:
assert "b" not in cache
assert "c" not in cache
+ def test_cache_update_existing_key_does_not_evict(self):
+ # Updating an existing key must not evict another entry or drop the
+ # cache below its limit (regression: it used to evict the oldest key).
+ cache: LocalCache[str, int] = LocalCache(limit=2)
+ cache["a"] = 1
+ cache["b"] = 2
+ cache["b"] = 20
+ assert len(cache) == 2
+ assert "a" in cache
+ assert cache["a"] == 1
+ assert cache["b"] == 20
+
class TestLocalWeakReferencedCache:
def test_cache_with_limit(self):