Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)

Andrey Rakhmatullin5a7e132486f3
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index be5efba87..db4e41c6a 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -151,7 +151,10 @@ class Downloader:
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
- def _get_slot(self, request: Request) -> tuple[str, Slot]:
+ @_warn_spider_arg
+ def _get_slot(
+ self, request: Request, spider: Spider | None = None
+ ) -> tuple[str, Slot]:
key = self.get_slot_key(request)
if key not in self.slots:
assert self.crawler.spider
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index ff9604c36..5d88b59c1 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacks
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -108,6 +109,24 @@ def test_params():
)
+def test_get_slot_deprecated_spider_arg():
+ crawler = get_crawler(DefaultSpider)
+ crawler.spider = crawler._create_spider()
+ downloader = Downloader(crawler)
+ downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
+ request = Request("https://example.com")
+
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",
+ ):
+ key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
+ key2, slot2 = downloader._get_slot(request)
+
+ assert key1 == key2
+ assert slot1 == slot2
+
+
@pytest.mark.parametrize(
"priority_queue_class",
[