Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let spiders change allowed_domains at run time (#7912)

Adrian0e324f3d4a55
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8fbf0c52d..f2cfeb712 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -59,9 +59,16 @@ scrapy.Spider
:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
enabled.
+ .. versionchanged:: VERSION
+ Changes to this attribute during a crawl are now taken into account.
+
Let's say your target url is ``https://www.example.com/1.html``,
then add ``'example.com'`` to the list.
+ You may modify this attribute while the spider runs, e.g. to allow
+ domains that you only learn about from an earlier response. The change
+ affects requests scheduled after it.
+
.. autoattribute:: start_urls
.. attribute:: custom_settings
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index db85b62a1..28c0e09cb 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -22,10 +22,12 @@ logger = logging.getLogger(__name__)
class OffsiteMiddleware:
crawler: Crawler
+ host_regex: re.Pattern[str]
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen: set[str] = set()
+ self._allowed_domains: list[str] | None = None
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@@ -37,7 +39,13 @@ class OffsiteMiddleware:
return o
def spider_opened(self, spider: Spider) -> None:
- self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
+ self._update_host_regex(spider)
+
+ def _update_host_regex(self, spider: Spider) -> None:
+ allowed_domains = list(getattr(spider, "allowed_domains", None) or [])
+ if allowed_domains != self._allowed_domains:
+ self._allowed_domains = allowed_domains
+ self.host_regex = self.get_host_regex(spider)
def request_scheduled(self, request: Request, spider: Spider) -> None:
self.process_request(request)
@@ -64,6 +72,7 @@ class OffsiteMiddleware:
raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
def should_follow(self, request: Request, spider: Spider) -> bool:
+ self._update_host_regex(spider)
regex = self.host_regex
# hostname can be None for wrong urls (like javascript links)
host = urlparse_cached(request).hostname or ""
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index cb17c2553..bab89814f 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -247,3 +247,50 @@ def test_ignore_request_reason():
IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")
):
mw.process_request(request)
+
+
+class DomainSpider(Spider):
+ name = "a"
+ allowed_domains: list[str]
+
+
+def test_dynamic_allowed_domains():
+ crawler = get_crawler(DomainSpider)
+ spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
+ crawler.spider = spider
+ mw = OffsiteMiddleware.from_crawler(crawler)
+ mw.spider_opened(spider)
+
+ with pytest.raises(IgnoreRequest):
+ mw.process_request(Request("https://b.example"))
+
+ spider.allowed_domains.append("b.example")
+ assert mw.process_request(Request("https://b.example")) is None
+
+ spider.allowed_domains.remove("a.example")
+ with pytest.raises(IgnoreRequest):
+ mw.process_request(Request("https://a.example"))
+
+
+def test_dynamic_allowed_domains_caching():
+ calls = 0
+
+ class TrackingMiddleware(OffsiteMiddleware):
+ def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
+ nonlocal calls
+ calls += 1
+ return super().get_host_regex(spider)
+
+ crawler = get_crawler(DomainSpider)
+ spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
+ crawler.spider = spider
+ mw = TrackingMiddleware.from_crawler(crawler)
+ mw.spider_opened(spider)
+
+ for _ in range(3):
+ mw.process_request(Request("https://a.example"))
+ assert calls == 1
+
+ spider.allowed_domains.append("b.example")
+ assert mw.process_request(Request("https://b.example")) is None
+ assert calls == 2