Scrapy, a fast high-level web crawling & scraping framework for Python.
new `allow_offsite` parameter in OffsiteMiddleware (#6151)
Kevin Lloyd Bernal1fc91bb46262
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex af7885a45..1ab8f588f 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -797,9 +797,12 @@ OffsiteMiddleware:attr:`~scrapy.Spider.allowed_domains` attribute, or theattribute is empty, the offsite middleware will allow all requests.- If the request has the :attr:`~scrapy.Request.dont_filter` attribute- set, the offsite middleware will allow the request even if its domain is not- listed in allowed domains.+ .. reqmeta:: allow_offsite++ If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to+ ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then+ the OffsiteMiddleware will allow the request even if its domain is not listed+ in allowed domains.RedirectMiddleware------------------diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 1bb1a10a4..b187f3aaf 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -145,9 +145,9 @@ Request objects:type priority: int:param dont_filter: indicates that this request should not be filtered by- the scheduler. This is used when you want to perform an identical- request multiple times, to ignore the duplicates filter. Use it with- care, or you will get into crawling loops. Default to ``False``.+ the scheduler or some middlewares. This is used when you want to perform+ an identical request multiple times, to ignore the duplicates filter.+ Use it with care, or you will get into crawling loops. Default to ``False``.:type dont_filter: bool:param errback: a function that will be called if any exception was@@ -660,6 +660,7 @@ are some special keys recognized by Scrapy and its built-in extensions.Those are:+* :reqmeta:`allow_offsite`* :reqmeta:`autothrottle_dont_adjust_delay`* :reqmeta:`bindaddress`* :reqmeta:`cookiejar`diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex a69f531a7..a2cff65e7 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -40,7 +40,11 @@ class OffsiteMiddleware:self.process_request(request, spider)def process_request(self, request: Request, spider: Spider) -> None:- if request.dont_filter or self.should_follow(request, spider):+ if (+ request.dont_filter+ or request.meta.get("allow_offsite")+ or self.should_follow(request, spider)+ ):returndomain = urlparse_cached(request).hostnameif domain and domain not in self.domains_seen:diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex d3ed64ef5..95e753830 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -61,7 +61,11 @@ class OffsiteMiddleware:def _filter(self, request: Any, spider: Spider) -> bool:if not isinstance(request, Request):return True- if request.dont_filter or self.should_follow(request, spider):+ if (+ request.dont_filter+ or request.meta.get("allow_offsite")+ or self.should_follow(request, spider)+ ):return Truedomain = urlparse_cached(request).hostnameif domain and domain not in self.domains_seen:diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex fec56a39f..23a1d06da 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -64,6 +64,37 @@ def test_process_request_dont_filter(value, filtered):assert mw.process_request(request, spider) is None+ ("allow_offsite", "dont_filter", "filtered"),+ (+ (True, UNSET, False),+ (True, None, False),+ (True, False, False),+ (True, True, False),+ (False, UNSET, True),+ (False, None, True),+ (False, False, True),+ (False, True, False),+ ),+)+def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):+ crawler = get_crawler(Spider)+ spider = crawler._create_spider(name="a", allowed_domains=["a.example"])+ mw = OffsiteMiddleware.from_crawler(crawler)+ mw.spider_opened(spider)+ kwargs = {"meta": {}}+ if allow_offsite is not UNSET:+ kwargs["meta"]["allow_offsite"] = allow_offsite+ if dont_filter is not UNSET:+ kwargs["dont_filter"] = dont_filter+ request = Request("https://b.example", **kwargs)+ if filtered:+ with pytest.raises(IgnoreRequest):+ mw.process_request(request, spider)+ else:+ assert mw.process_request(request, spider) is None++@pytest.mark.parametrize("value",(diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 837f1c2c8..906928e01 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -29,6 +29,7 @@ class TestOffsiteMiddleware(TestCase):Request("http://scrapy.org/1"),Request("http://sub.scrapy.org/1"),Request("http://offsite.tld/letmepass", dont_filter=True),+ Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}),Request("http://scrapy.test.org/"),Request("http://scrapy.test.org:8000/"),]