Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix scrapy.utils.datatypes.LocalCache limit issue
Eugenio Lacuesta439a3e59b8e8
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex df2b99c28..f7e3240c1 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict):self.limit = limitdef __setitem__(self, key, value):- while len(self) >= self.limit:- self.popitem(last=False)+ if self.limit:+ while len(self) >= self.limit:+ self.popitem(last=False)super(LocalCache, self).__setitem__(key, value)diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 535095b8d..6ffd7c73c 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -7,7 +7,7 @@ if six.PY2:else:from collections.abc import Mapping, MutableMapping-from scrapy.utils.datatypes import CaselessDict, SequenceExclude+from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache__doctests__ = ['scrapy.utils.datatypes']@@ -242,6 +242,31 @@ class SequenceExcludeTest(unittest.TestCase):for v in [-3, "test", 1.1]:self.assertNotIn(v, d)++class LocalCacheTest(unittest.TestCase):++ def test_cache_with_limit(self):+ cache = LocalCache(limit=2)+ cache['a'] = 1+ cache['b'] = 2+ cache['c'] = 3+ self.assertEqual(len(cache), 2)+ self.assertNotIn('a', cache)+ self.assertIn('b', cache)+ self.assertIn('c', cache)+ self.assertEqual(cache['b'], 2)+ self.assertEqual(cache['c'], 3)++ def test_cache_without_limit(self):+ max = 10**4+ cache = LocalCache()+ for x in range(max):+ cache[str(x)] = x+ self.assertEqual(len(cache), max)+ for x in range(max):+ self.assertIn(str(x), cache)+ self.assertEqual(cache[str(x)], x)++if __name__ == "__main__":unittest.main()-