Scrapy, a fast high-level web crawling & scraping framework for Python.
Re-resolve the download slot of redirects and retries to another domain (#8337)
Adrian78f4b3f17195
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 3f5c189af..f512c722a 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1189,8 +1189,13 @@ to a specific slot instead, set the name of the slot as the ``download_slot``:attr:`Request.meta <scrapy.Request.meta>` key. Once a request is assigned to aslot, that key holds the name of the slot.-Since that key is kept on redirects, a redirected request stays in the slot of-the request it comes from, even when it points to a different domain.+Redirects and retries are assigned to a slot based on their own URL domain,+unless the slot of their original request was set through the ``download_slot``+key, in which case they keep that slot.++.. versionchanged:: VERSION+ In earlier versions, redirects and retries always kept the slot of their+ original request, even when they pointed to a different domain... setting:: DOWNLOAD_TIMEOUTdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 54f018d13..ba9d6afe2 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -176,6 +176,9 @@ def _slot_jitter(slot_settings: dict[str, Any], default: float) -> float:return default+_STAMPED_SLOT_META_KEY = "_download_slot_stamp"++class Downloader:DOWNLOAD_SLOT = "download_slot"_SLOT_GC_INTERVAL: float = 60.0 # seconds@@ -255,7 +258,9 @@ class Downloader:def get_slot_key(self, request: Request) -> str:meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)- if meta_slot is not None:+ if meta_slot is not None and meta_slot != request.meta.get(+ _STAMPED_SLOT_META_KEY+ ):return meta_slotkey = urlparse_cached(request).hostname or ""@@ -271,6 +276,13 @@ class Downloader:"The downloader is shutting down and not accepting new requests")key, slot = self._get_slot(request)+ # Requests derived from this one (redirects, retries) copy its meta.+ # The stamp tells a slot the downloader assigned, which they must+ # resolve again for their own URL, from one set by the user, which+ # they keep.+ previous: str | None = request.meta.get(self.DOWNLOAD_SLOT)+ if previous is None or previous == request.meta.get(_STAMPED_SLOT_META_KEY):+ request.meta[_STAMPED_SLOT_META_KEY] = keyrequest.meta[self.DOWNLOAD_SLOT] = keyslot.active.add(request)self.signals.send_catch_log(diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 8a805ada2..69e334e94 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -174,3 +174,34 @@ async def test_none_slot_with_priority_queue(stats = crawler.statsassert stats.get_value("spider_exceptions", 0) == 0assert stats.get_value("downloader/exception_count", 0) == 0+++class CrossDomainRedirectSpider(MetaSpider):+ name = "cross_domain_redirect"++ def __init__(self, *args: Any, slot: str | None = None, **kwargs: Any):+ super().__init__(*args, **kwargs)+ self.slot = slot++ async def start(self):+ assert self.mockserver+ target = self.mockserver.url("/text").replace("127.0.0.1", "localhost")+ url = self.mockserver.url(f"/redirect-to?goto={target}")+ meta = {} if self.slot is None else {"download_slot": self.slot}+ yield Request(url, meta=meta, callback=self.parse)++ def parse(self, response):+ self.meta["slot"] = response.meta["download_slot"]+++ ("slot", "expected"), [(None, "localhost"), ("custom", "custom")]+)+@coroutine_test+async def test_cross_domain_redirect(+ mockserver: MockServer, slot: str | None, expected: str+) -> None:+ crawler = get_crawler(CrossDomainRedirectSpider)+ await crawler.crawl_async(mockserver=mockserver, slot=slot)+ assert isinstance(crawler.spider, CrossDomainRedirectSpider)+ assert crawler.spider.meta["slot"] == expected