Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Pass the spider arg to custom stat collectors {open,close}_spider(). (#7213)

Andrey Rakhmatullin0f07b2e38c0b
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4f8d8e8a1..b29c5e80e 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -534,7 +534,16 @@ class ExecutionEngine:
await maybe_deferred_to_future(d)
await self.scraper.open_spider_async()
assert self.crawler.stats
- self.crawler.stats.open_spider()
+ if argument_is_required(self.crawler.stats.open_spider, "spider"):
+ warnings.warn(
+ f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+ f" this is deprecated and the argument will not be passed in future Scrapy versions.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ self.crawler.stats.open_spider(spider=self.crawler.spider)
+ else:
+ self.crawler.stats.open_spider()
await self.signals.send_catch_log_async(
signals.spider_opened, spider=self.crawler.spider
)
@@ -629,7 +638,18 @@ class ExecutionEngine:
assert self.crawler.stats
try:
- self.crawler.stats.close_spider(reason=reason)
+ if argument_is_required(self.crawler.stats.close_spider, "spider"):
+ warnings.warn(
+ f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+ f" this is deprecated and the argument will not be passed in future Scrapy versions.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ self.crawler.stats.close_spider(
+ spider=self.crawler.spider, reason=reason
+ )
+ else:
+ self.crawler.stats.close_spider(reason=reason)
except Exception:
log_failure("Stats close failure")
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 6814a652e..b5ab06901 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -10,7 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
+from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
+from tests.spiders import SimpleSpider
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@@ -109,12 +111,77 @@ class TestStatsCollector:
stats = StatsCollector(crawler)
with pytest.warns(
ScrapyDeprecationWarning,
- match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated",
+ match=r"Passing a 'spider' argument to StatsCollector\.set_value\(\) is deprecated",
):
stats.set_value("test", "value", spider=spider)
assert stats.get_stats() == {"test": "value"}
with pytest.warns(
ScrapyDeprecationWarning,
- match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated",
+ match=r"Passing a 'spider' argument to StatsCollector\.get_stats\(\) is deprecated",
):
assert stats.get_stats(spider) == {"test": "value"}
+
+ @deferred_f_from_coro_f
+ async def test_deprecated_spider_arg_custom_collector(self) -> None:
+ class CustomStatsCollector:
+ def __init__(self, crawler):
+ self._stats = {}
+
+ def open_spider(self, spider):
+ pass
+
+ def get_stats(self, spider=None):
+ return self._stats
+
+ def inc_value(self, key, count=1, start=0, spider=None):
+ d = self._stats
+ d[key] = d.setdefault(key, start) + count
+
+ def close_spider(self, spider, reason):
+ pass
+
+ crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+ ),
+ ):
+ await crawler.crawl_async(url="data:,")
+
+ @deferred_f_from_coro_f
+ async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:
+ class CustomStatsCollector(StatsCollector):
+ def open_spider(self, spider): # pylint: disable=signature-differs
+ super().open_spider(spider)
+
+ def inc_value(self, key, count=1, start=0, spider=None): # pylint: disable=useless-parent-delegation
+ super().inc_value(key, count, start, spider)
+
+ def close_spider(self, spider, reason): # pylint: disable=signature-differs
+ super().close_spider(spider, reason)
+
+ crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a 'spider' argument to .*CustomStatsCollector\.open_spider\(\) is deprecated",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a 'spider' argument to .*CustomStatsCollector\.close_spider\(\) is deprecated",
+ ),
+ ):
+ await crawler.crawl_async(url="data:,")