Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Re-resolve the download slot of redirects and retries to another domain (#8337)

Adrian78f4b3f17195
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3f5c189af..f512c722a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1189,8 +1189,13 @@ to a specific slot instead, set the name of the slot as the ``download_slot``
:attr:`Request.meta <scrapy.Request.meta>` key. Once a request is assigned to a
slot, that key holds the name of the slot.
-Since that key is kept on redirects, a redirected request stays in the slot of
-the request it comes from, even when it points to a different domain.
+Redirects and retries are assigned to a slot based on their own URL domain,
+unless the slot of their original request was set through the ``download_slot``
+key, in which case they keep that slot.
+
+.. versionchanged:: VERSION
+ In earlier versions, redirects and retries always kept the slot of their
+ original request, even when they pointed to a different domain.
.. setting:: DOWNLOAD_TIMEOUT
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 54f018d13..ba9d6afe2 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -176,6 +176,9 @@ def _slot_jitter(slot_settings: dict[str, Any], default: float) -> float:
return default
+_STAMPED_SLOT_META_KEY = "_download_slot_stamp"
+
+
class Downloader:
DOWNLOAD_SLOT = "download_slot"
_SLOT_GC_INTERVAL: float = 60.0 # seconds
@@ -255,7 +258,9 @@ class Downloader:
def get_slot_key(self, request: Request) -> str:
meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
- if meta_slot is not None:
+ if meta_slot is not None and meta_slot != request.meta.get(
+ _STAMPED_SLOT_META_KEY
+ ):
return meta_slot
key = urlparse_cached(request).hostname or ""
@@ -271,6 +276,13 @@ class Downloader:
"The downloader is shutting down and not accepting new requests"
)
key, slot = self._get_slot(request)
+ # Requests derived from this one (redirects, retries) copy its meta.
+ # The stamp tells a slot the downloader assigned, which they must
+ # resolve again for their own URL, from one set by the user, which
+ # they keep.
+ previous: str | None = request.meta.get(self.DOWNLOAD_SLOT)
+ if previous is None or previous == request.meta.get(_STAMPED_SLOT_META_KEY):
+ request.meta[_STAMPED_SLOT_META_KEY] = key
request.meta[self.DOWNLOAD_SLOT] = key
slot.active.add(request)
self.signals.send_catch_log(
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 8a805ada2..69e334e94 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -174,3 +174,34 @@ async def test_none_slot_with_priority_queue(
stats = crawler.stats
assert stats.get_value("spider_exceptions", 0) == 0
assert stats.get_value("downloader/exception_count", 0) == 0
+
+
+class CrossDomainRedirectSpider(MetaSpider):
+ name = "cross_domain_redirect"
+
+ def __init__(self, *args: Any, slot: str | None = None, **kwargs: Any):
+ super().__init__(*args, **kwargs)
+ self.slot = slot
+
+ async def start(self):
+ assert self.mockserver
+ target = self.mockserver.url("/text").replace("127.0.0.1", "localhost")
+ url = self.mockserver.url(f"/redirect-to?goto={target}")
+ meta = {} if self.slot is None else {"download_slot": self.slot}
+ yield Request(url, meta=meta, callback=self.parse)
+
+ def parse(self, response):
+ self.meta["slot"] = response.meta["download_slot"]
+
+
+ ("slot", "expected"), [(None, "localhost"), ("custom", "custom")]
+)
+@coroutine_test
+async def test_cross_domain_redirect(
+ mockserver: MockServer, slot: str | None, expected: str
+) -> None:
+ crawler = get_crawler(CrossDomainRedirectSpider)
+ await crawler.crawl_async(mockserver=mockserver, slot=slot)
+ assert isinstance(crawler.spider, CrossDomainRedirectSpider)
+ assert crawler.spider.meta["slot"] == expected