Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Document how to disallow subdomains of allowed domains (#7903)

Adriand41bfaec05d9
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 965108214..dec9904d2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -799,40 +799,9 @@ OffsiteMiddleware
.. module:: scrapy.downloadermiddlewares.offsite
:synopsis: Offsite Middleware
-.. class:: OffsiteMiddleware
+.. autoclass:: OffsiteMiddleware
- .. versionadded:: 2.11.2
-
- Filters out Requests for URLs outside the domains covered by the spider.
-
- This middleware filters out every request whose host names aren't in the
- spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
- All subdomains of any domain in the list are also allowed.
- E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
- but not ``www2.example.com`` nor ``example.com``.
-
- When your spider returns a request for a domain not belonging to those
- covered by the spider, this middleware will log a debug message similar to
- this one::
-
- DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
-
- To avoid filling the log with too much noise, it will only print one of
- these messages for each new domain filtered. So, for example, if another
- request for ``offsite.example`` is filtered, no log message will be
- printed. But if a request for ``other.example`` is filtered, a message
- will be printed (but only for the first request filtered).
-
- If the spider doesn't define an
- :attr:`~scrapy.Spider.allowed_domains` attribute, or the
- attribute is empty, the offsite middleware will allow all requests.
-
- .. reqmeta:: allow_offsite
-
- If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
- ``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
- set to ``True``, then the OffsiteMiddleware will allow the request even if
- its domain is not listed in allowed domains.
+ .. automethod:: should_follow
RedirectMiddleware
------------------
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 28c0e09cb..0d5c9e4c9 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -21,6 +21,36 @@ logger = logging.getLogger(__name__)
class OffsiteMiddleware:
+ """Filter out requests for URLs outside the domains covered by the spider.
+
+ .. versionadded:: 2.11.2
+
+ A request is allowed if its host name is in the
+ :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a
+ subdomain of one of those domains. E.g. ``www.example.org`` also allows
+ ``bob.www.example.org``, but neither ``www2.example.org`` nor
+ ``example.org``. See :meth:`should_follow` to use a different policy.
+
+ If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or
+ the attribute is empty, every request is allowed.
+
+ Filtered requests are logged as follows::
+
+ DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
+
+ Only the first request filtered for a given domain is logged, to keep the
+ log readable.
+
+ .. reqmeta:: allow_offsite
+
+ allow_offsite
+ -------------
+
+ Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to
+ ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are
+ allowed regardless of their host name.
+ """
+
crawler: Crawler
host_regex: re.Pattern[str]
@@ -72,6 +102,23 @@ class OffsiteMiddleware:
raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
def should_follow(self, request: Request, spider: Spider) -> bool:
+ """Return ``True`` if *request* is on site, ``False`` if it must be
+ filtered out.
+
+ Override this method to implement a different offsite policy. For
+ example, to allow the domains in
+ :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains:
+
+ .. code-block:: python
+
+ from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
+ from scrapy.utils.httpobj import urlparse_cached
+
+
+ class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
+ def should_follow(self, request, spider):
+ return urlparse_cached(request).hostname in spider.allowed_domains
+ """
self._update_host_regex(spider)
regex = self.host_regex
# hostname can be None for wrong urls (like javascript links)
@@ -79,7 +126,6 @@ class OffsiteMiddleware:
return bool(regex.search(host))
def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
- """Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, "allowed_domains", None)
if not allowed_domains:
return re.compile("") # allow all by default
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index bab89814f..1f91dcd4b 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -6,6 +6,7 @@ import pytest
from scrapy import Request, Spider
from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.test import get_crawler
UNSET = object()
@@ -237,6 +238,21 @@ def test_repeated_offsite_domain():
assert crawler.stats.get_value("offsite/filtered") == 2
+def test_should_follow_override():
+ class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
+ def should_follow(self, request: Request, spider: Spider) -> bool:
+ allowed_domains: list[str] = getattr(spider, "allowed_domains", [])
+ return urlparse_cached(request).hostname in allowed_domains
+
+ crawler = get_crawler(Spider)
+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
+ mw = RootOnlyOffsiteMiddleware.from_crawler(crawler)
+ mw.spider_opened(crawler.spider)
+ assert mw.process_request(Request("https://example.com/1")) is None
+ with pytest.raises(IgnoreRequest):
+ mw.process_request(Request("https://www.example.com/1"))
+
+
def test_ignore_request_reason():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])