Scrapy, a fast high-level web crawling & scraping framework for Python.
Pass the spider arg to custom stat collectors {open,close}_spider(). (#7213)
Andrey Rakhmatullin0f07b2e38c0b
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 4f8d8e8a1..b29c5e80e 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -534,7 +534,16 @@ class ExecutionEngine:await maybe_deferred_to_future(d)await self.scraper.open_spider_async()assert self.crawler.stats- self.crawler.stats.open_spider()+ if argument_is_required(self.crawler.stats.open_spider, "spider"):+ warnings.warn(+ f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"+ f" this is deprecated and the argument will not be passed in future Scrapy versions.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ self.crawler.stats.open_spider(spider=self.crawler.spider)+ else:+ self.crawler.stats.open_spider()await self.signals.send_catch_log_async(signals.spider_opened, spider=self.crawler.spider)@@ -629,7 +638,18 @@ class ExecutionEngine:assert self.crawler.statstry:- self.crawler.stats.close_spider(reason=reason)+ if argument_is_required(self.crawler.stats.close_spider, "spider"):+ warnings.warn(+ f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"+ f" this is deprecated and the argument will not be passed in future Scrapy versions.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ self.crawler.stats.close_spider(+ spider=self.crawler.spider, reason=reason+ )+ else:+ self.crawler.stats.close_spider(reason=reason)except Exception:log_failure("Stats close failure")diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 6814a652e..b5ab06901 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -10,7 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.corestats import CoreStatsfrom scrapy.spiders import Spiderfrom scrapy.statscollectors import DummyStatsCollector, StatsCollector+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawler+from tests.spiders import SimpleSpiderif TYPE_CHECKING:from scrapy.crawler import Crawler@@ -109,12 +111,77 @@ class TestStatsCollector:stats = StatsCollector(crawler)with pytest.warns(ScrapyDeprecationWarning,- match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated",+ match=r"Passing a 'spider' argument to StatsCollector\.set_value\(\) is deprecated",):stats.set_value("test", "value", spider=spider)assert stats.get_stats() == {"test": "value"}with pytest.warns(ScrapyDeprecationWarning,- match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated",+ match=r"Passing a 'spider' argument to StatsCollector\.get_stats\(\) is deprecated",):assert stats.get_stats(spider) == {"test": "value"}++ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg_custom_collector(self) -> None:+ class CustomStatsCollector:+ def __init__(self, crawler):+ self._stats = {}++ def open_spider(self, spider):+ pass++ def get_stats(self, spider=None):+ return self._stats++ def inc_value(self, key, count=1, start=0, spider=None):+ d = self._stats+ d[key] = d.setdefault(key, start) + count++ def close_spider(self, spider, reason):+ pass++ crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",+ ),+ ):+ await crawler.crawl_async(url="data:,")++ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:+ class CustomStatsCollector(StatsCollector):+ def open_spider(self, spider): # pylint: disable=signature-differs+ super().open_spider(spider)++ def inc_value(self, key, count=1, start=0, spider=None): # pylint: disable=useless-parent-delegation+ super().inc_value(key, count, start, spider)++ def close_spider(self, spider, reason): # pylint: disable=signature-differs+ super().close_spider(spider, reason)++ crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a 'spider' argument to .*CustomStatsCollector\.open_spider\(\) is deprecated",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a 'spider' argument to .*CustomStatsCollector\.close_spider\(\) is deprecated",+ ),+ ):+ await crawler.crawl_async(url="data:,")