Scrapy, a fast high-level web crawling & scraping framework for Python.
Let spiders change allowed_domains at run time (#7912)
Adrian0e324f3d4a55
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 8fbf0c52d..f2cfeb712 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -59,9 +59,16 @@ scrapy.Spider:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` isenabled.+ .. versionchanged:: VERSION+ Changes to this attribute during a crawl are now taken into account.+Let's say your target url is ``https://www.example.com/1.html``,then add ``'example.com'`` to the list.+ You may modify this attribute while the spider runs, e.g. to allow+ domains that you only learn about from an earlier response. The change+ affects requests scheduled after it.+.. autoattribute:: start_urls.. attribute:: custom_settingsdiff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex db85b62a1..28c0e09cb 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -22,10 +22,12 @@ logger = logging.getLogger(__name__)class OffsiteMiddleware:crawler: Crawler+ host_regex: re.Pattern[str]def __init__(self, stats: StatsCollector):self.stats = statsself.domains_seen: set[str] = set()+ self._allowed_domains: list[str] | None = None@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:@@ -37,7 +39,13 @@ class OffsiteMiddleware:return odef spider_opened(self, spider: Spider) -> None:- self.host_regex: re.Pattern[str] = self.get_host_regex(spider)+ self._update_host_regex(spider)++ def _update_host_regex(self, spider: Spider) -> None:+ allowed_domains = list(getattr(spider, "allowed_domains", None) or [])+ if allowed_domains != self._allowed_domains:+ self._allowed_domains = allowed_domains+ self.host_regex = self.get_host_regex(spider)def request_scheduled(self, request: Request, spider: Spider) -> None:self.process_request(request)@@ -64,6 +72,7 @@ class OffsiteMiddleware:raise IgnoreRequest(f"Filtered offsite request to {domain!r}")def should_follow(self, request: Request, spider: Spider) -> bool:+ self._update_host_regex(spider)regex = self.host_regex# hostname can be None for wrong urls (like javascript links)host = urlparse_cached(request).hostname or ""diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex cb17c2553..bab89814f 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -247,3 +247,50 @@ def test_ignore_request_reason():IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")):mw.process_request(request)+++class DomainSpider(Spider):+ name = "a"+ allowed_domains: list[str]+++def test_dynamic_allowed_domains():+ crawler = get_crawler(DomainSpider)+ spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])+ crawler.spider = spider+ mw = OffsiteMiddleware.from_crawler(crawler)+ mw.spider_opened(spider)++ with pytest.raises(IgnoreRequest):+ mw.process_request(Request("https://b.example"))++ spider.allowed_domains.append("b.example")+ assert mw.process_request(Request("https://b.example")) is None++ spider.allowed_domains.remove("a.example")+ with pytest.raises(IgnoreRequest):+ mw.process_request(Request("https://a.example"))+++def test_dynamic_allowed_domains_caching():+ calls = 0++ class TrackingMiddleware(OffsiteMiddleware):+ def get_host_regex(self, spider: Spider) -> re.Pattern[str]:+ nonlocal calls+ calls += 1+ return super().get_host_regex(spider)++ crawler = get_crawler(DomainSpider)+ spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])+ crawler.spider = spider+ mw = TrackingMiddleware.from_crawler(crawler)+ mw.spider_opened(spider)++ for _ in range(3):+ mw.process_request(Request("https://a.example"))+ assert calls == 1++ spider.allowed_domains.append("b.example")+ assert mw.process_request(Request("https://b.example")) is None+ assert calls == 2