Scrapy, a fast high-level web crawling & scraping framework for Python.
Make stats collectors mutable mappings (#8241)
Adrian2a3f22b448ca
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rstindex 6d61f283b..ea80a4872 100644--- a/docs/topics/stats.rst+++ b/docs/topics/stats.rst@@ -82,6 +82,19 @@ Get all stats:>>> stats.get_stats(){'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}+Stats collectors that subclass :class:`StatsCollector`, like the built-in ones,+are also :class:`mutable mappings <collections.abc.MutableMapping>` of stats:++.. versionadded:: VERSION++.. code-block:: python++ stats["hostname"] = socket.gethostname()+ hostname = stats.get("hostname")++As with a :class:`dict`, ``stats[key]`` raises :exc:`KeyError` for a missing+stat, and with :class:`DummyStatsCollector` every stat is missing.+.. skip: endAvailable Stats Collectorsdiff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.pyindex b1ce467e9..8171907d5 100644--- a/scrapy/statscollectors.py+++ b/scrapy/statscollectors.py@@ -6,11 +6,14 @@ from __future__ import annotationsimport loggingimport pprint+from collections.abc import MutableMappingfrom typing import TYPE_CHECKING, Any, TypeAliasfrom scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:+ from collections.abc import Iterator+from scrapy import Spiderfrom scrapy.crawler import Crawler@@ -20,10 +23,15 @@ logger = logging.getLogger(__name__)StatsT: TypeAlias = dict[str, Any]+_MISSING = object()+-class StatsCollector:+class StatsCollector(MutableMapping[str, Any]):"""The base stats collector that other stats collectors are based on."""+ __eq__ = object.__eq__+ __hash__ = object.__hash__+def __init__(self, crawler: Crawler):self._dump: bool = crawler.settings.getbool("STATS_DUMP")self._stats: StatsT = {}@@ -116,6 +124,28 @@ class StatsCollector:def __str__(self) -> str:return pprint.pformat(self._stats)+ # Components use ``if stats:`` to tell a stats collector from None.+ def __bool__(self) -> bool:+ return True++ def __getitem__(self, key: str) -> Any:+ value = self.get_value(key, _MISSING)+ if value is _MISSING:+ raise KeyError(key)+ return value++ def __setitem__(self, key: str, value: Any) -> None:+ self.set_value(key, value)++ def __delitem__(self, key: str) -> None:+ del self._stats[key]++ def __iter__(self) -> Iterator[str]:+ return iter(self.get_stats())++ def __len__(self) -> int:+ return len(self.get_stats())+class MemoryStatsCollector(StatsCollector):"""A simple stats collector that keeps the stats of the last scraping rundiff --git a/tests/test_stats.py b/tests/test_stats.pyindex 53b2d09cd..3cc9b2186 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -121,6 +121,42 @@ class TestStatsCollector:assert stats.get_stats() == {}stats.close_spider()+ def test_mapping(self, crawler: Crawler) -> None:+ stats = StatsCollector(crawler)+ assert stats+ assert len(stats) == 0+ with pytest.raises(KeyError):+ stats["a"]+ stats["a"] = 1+ stats.update(b=2)+ assert stats["a"] == 1+ assert "a" in stats+ assert list(stats) == ["a", "b"]+ assert len(stats) == 2+ del stats["a"]+ assert stats.get_stats() == {"b": 2}+ with pytest.raises(KeyError):+ del stats["a"]++ def test_mapping_identity(self, crawler: Crawler) -> None:+ stats1 = StatsCollector(crawler)+ stats2 = StatsCollector(crawler)+ assert stats1 != stats2+ assert stats1 != {}+ assert len({stats1, stats2}) == 2++ def test_dummy_mapping(self) -> None:+ crawler = get_crawler(Spider, {"STATS_DUMP": False})+ stats = DummyStatsCollector(crawler)+ stats["a"] = 1+ assert stats+ assert len(stats) == 0+ assert "a" not in stats+ with pytest.raises(KeyError):+ stats["a"]+ with pytest.raises(KeyError):+ del stats["a"]+def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:stats = StatsCollector(crawler)with pytest.warns(