Scrapy, a fast high-level web crawling & scraping framework for Python.
Document how to disallow subdomains of allowed domains (#7903)
Adriand41bfaec05d9
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 965108214..dec9904d2 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -799,40 +799,9 @@ OffsiteMiddleware.. module:: scrapy.downloadermiddlewares.offsite:synopsis: Offsite Middleware-.. class:: OffsiteMiddleware+.. autoclass:: OffsiteMiddleware- .. versionadded:: 2.11.2-- Filters out Requests for URLs outside the domains covered by the spider.-- This middleware filters out every request whose host names aren't in the- spider's :attr:`~scrapy.Spider.allowed_domains` attribute.- All subdomains of any domain in the list are also allowed.- E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``- but not ``www2.example.com`` nor ``example.com``.-- When your spider returns a request for a domain not belonging to those- covered by the spider, this middleware will log a debug message similar to- this one::-- DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>-- To avoid filling the log with too much noise, it will only print one of- these messages for each new domain filtered. So, for example, if another- request for ``offsite.example`` is filtered, no log message will be- printed. But if a request for ``other.example`` is filtered, a message- will be printed (but only for the first request filtered).-- If the spider doesn't define an- :attr:`~scrapy.Spider.allowed_domains` attribute, or the- attribute is empty, the offsite middleware will allow all requests.-- .. reqmeta:: allow_offsite-- If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to- ``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``- set to ``True``, then the OffsiteMiddleware will allow the request even if- its domain is not listed in allowed domains.+ .. automethod:: should_followRedirectMiddleware------------------diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex 28c0e09cb..0d5c9e4c9 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -21,6 +21,36 @@ logger = logging.getLogger(__name__)class OffsiteMiddleware:+ """Filter out requests for URLs outside the domains covered by the spider.++ .. versionadded:: 2.11.2++ A request is allowed if its host name is in the+ :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a+ subdomain of one of those domains. E.g. ``www.example.org`` also allows+ ``bob.www.example.org``, but neither ``www2.example.org`` nor+ ``example.org``. See :meth:`should_follow` to use a different policy.++ If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or+ the attribute is empty, every request is allowed.++ Filtered requests are logged as follows::++ DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>++ Only the first request filtered for a given domain is logged, to keep the+ log readable.++ .. reqmeta:: allow_offsite++ allow_offsite+ -------------++ Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to+ ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are+ allowed regardless of their host name.+ """+crawler: Crawlerhost_regex: re.Pattern[str]@@ -72,6 +102,23 @@ class OffsiteMiddleware:raise IgnoreRequest(f"Filtered offsite request to {domain!r}")def should_follow(self, request: Request, spider: Spider) -> bool:+ """Return ``True`` if *request* is on site, ``False`` if it must be+ filtered out.++ Override this method to implement a different offsite policy. For+ example, to allow the domains in+ :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains:++ .. code-block:: python++ from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware+ from scrapy.utils.httpobj import urlparse_cached+++ class RootOnlyOffsiteMiddleware(OffsiteMiddleware):+ def should_follow(self, request, spider):+ return urlparse_cached(request).hostname in spider.allowed_domains+ """self._update_host_regex(spider)regex = self.host_regex# hostname can be None for wrong urls (like javascript links)@@ -79,7 +126,6 @@ class OffsiteMiddleware:return bool(regex.search(host))def get_host_regex(self, spider: Spider) -> re.Pattern[str]:- """Override this method to implement a different offsite policy"""allowed_domains = getattr(spider, "allowed_domains", None)if not allowed_domains:return re.compile("") # allow all by defaultdiff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex bab89814f..1f91dcd4b 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -6,6 +6,7 @@ import pytestfrom scrapy import Request, Spiderfrom scrapy.downloadermiddlewares.offsite import OffsiteMiddlewarefrom scrapy.exceptions import IgnoreRequest+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.test import get_crawlerUNSET = object()@@ -237,6 +238,21 @@ def test_repeated_offsite_domain():assert crawler.stats.get_value("offsite/filtered") == 2+def test_should_follow_override():+ class RootOnlyOffsiteMiddleware(OffsiteMiddleware):+ def should_follow(self, request: Request, spider: Spider) -> bool:+ allowed_domains: list[str] = getattr(spider, "allowed_domains", [])+ return urlparse_cached(request).hostname in allowed_domains++ crawler = get_crawler(Spider)+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])+ mw = RootOnlyOffsiteMiddleware.from_crawler(crawler)+ mw.spider_opened(crawler.spider)+ assert mw.process_request(Request("https://example.com/1")) is None+ with pytest.raises(IgnoreRequest):+ mw.process_request(Request("https://www.example.com/1"))++def test_ignore_request_reason():crawler = get_crawler(Spider)crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])