Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make stats collectors mutable mappings (#8241)

Adrian2a3f22b448ca
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 6d61f283b..ea80a4872 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -82,6 +82,19 @@ Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+Stats collectors that subclass :class:`StatsCollector`, like the built-in ones,
+are also :class:`mutable mappings <collections.abc.MutableMapping>` of stats:
+
+.. versionadded:: VERSION
+
+.. code-block:: python
+
+ stats["hostname"] = socket.gethostname()
+ hostname = stats.get("hostname")
+
+As with a :class:`dict`, ``stats[key]`` raises :exc:`KeyError` for a missing
+stat, and with :class:`DummyStatsCollector` every stat is missing.
+
.. skip: end
Available Stats Collectors
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index b1ce467e9..8171907d5 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -6,11 +6,14 @@ from __future__ import annotations
import logging
import pprint
+from collections.abc import MutableMapping
from typing import TYPE_CHECKING, Any, TypeAlias
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
+ from collections.abc import Iterator
+
from scrapy import Spider
from scrapy.crawler import Crawler
@@ -20,10 +23,15 @@ logger = logging.getLogger(__name__)
StatsT: TypeAlias = dict[str, Any]
+_MISSING = object()
+
-class StatsCollector:
+class StatsCollector(MutableMapping[str, Any]):
"""The base stats collector that other stats collectors are based on."""
+ __eq__ = object.__eq__
+ __hash__ = object.__hash__
+
def __init__(self, crawler: Crawler):
self._dump: bool = crawler.settings.getbool("STATS_DUMP")
self._stats: StatsT = {}
@@ -116,6 +124,28 @@ class StatsCollector:
def __str__(self) -> str:
return pprint.pformat(self._stats)
+ # Components use ``if stats:`` to tell a stats collector from None.
+ def __bool__(self) -> bool:
+ return True
+
+ def __getitem__(self, key: str) -> Any:
+ value = self.get_value(key, _MISSING)
+ if value is _MISSING:
+ raise KeyError(key)
+ return value
+
+ def __setitem__(self, key: str, value: Any) -> None:
+ self.set_value(key, value)
+
+ def __delitem__(self, key: str) -> None:
+ del self._stats[key]
+
+ def __iter__(self) -> Iterator[str]:
+ return iter(self.get_stats())
+
+ def __len__(self) -> int:
+ return len(self.get_stats())
+
class MemoryStatsCollector(StatsCollector):
"""A simple stats collector that keeps the stats of the last scraping run
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 53b2d09cd..3cc9b2186 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -121,6 +121,42 @@ class TestStatsCollector:
assert stats.get_stats() == {}
stats.close_spider()
+ def test_mapping(self, crawler: Crawler) -> None:
+ stats = StatsCollector(crawler)
+ assert stats
+ assert len(stats) == 0
+ with pytest.raises(KeyError):
+ stats["a"]
+ stats["a"] = 1
+ stats.update(b=2)
+ assert stats["a"] == 1
+ assert "a" in stats
+ assert list(stats) == ["a", "b"]
+ assert len(stats) == 2
+ del stats["a"]
+ assert stats.get_stats() == {"b": 2}
+ with pytest.raises(KeyError):
+ del stats["a"]
+
+ def test_mapping_identity(self, crawler: Crawler) -> None:
+ stats1 = StatsCollector(crawler)
+ stats2 = StatsCollector(crawler)
+ assert stats1 != stats2
+ assert stats1 != {}
+ assert len({stats1, stats2}) == 2
+
+ def test_dummy_mapping(self) -> None:
+ crawler = get_crawler(Spider, {"STATS_DUMP": False})
+ stats = DummyStatsCollector(crawler)
+ stats["a"] = 1
+ assert stats
+ assert len(stats) == 0
+ assert "a" not in stats
+ with pytest.raises(KeyError):
+ stats["a"]
+ with pytest.raises(KeyError):
+ del stats["a"]
+
def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:
stats = StatsCollector(crawler)
with pytest.warns(