Scrapy, a fast high-level web crawling & scraping framework for Python.
Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)
Andrey Rakhmatullin5a7e132486f3
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex be5efba87..db4e41c6a 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -151,7 +151,10 @@ class Downloader:def needs_backout(self) -> bool:return len(self.active) >= self.total_concurrency- def _get_slot(self, request: Request) -> tuple[str, Slot]:+ @_warn_spider_arg+ def _get_slot(+ self, request: Request, spider: Spider | None = None+ ) -> tuple[str, Slot]:key = self.get_slot_key(request)if key not in self.slots:assert self.crawler.spiderdiff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex ff9604c36..5d88b59c1 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacksfrom scrapy import Requestfrom scrapy.core.downloader import Downloader, Slotfrom scrapy.crawler import CrawlerRunner+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -108,6 +109,24 @@ def test_params():)+def test_get_slot_deprecated_spider_arg():+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ downloader = Downloader(crawler)+ downloader._slot_gc_loop.stop() # Prevent an unclean reactor.+ request = Request("https://example.com")++ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",+ ):+ key1, slot1 = downloader._get_slot(request, spider=crawler.spider)+ key2, slot2 = downloader._get_slot(request)++ assert key1 == key2+ assert slot1 == slot2++@pytest.mark.parametrize("priority_queue_class",[