Scrapy, a fast high-level web crawling & scraping framework for Python.
Base class for universal spider middlewares (#6693)
Andrey Rakhmatullindaf9db72b2dc
diff --git a/docs/news.rst b/docs/news.rstindex 9a68f8852..9f476ee21 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,23 @@Release notes=============+.. _release-VERSION:++Scrapy VERSION (unreleased)+---------------------------++Backward-incompatible changes+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++- The ``from_settings()`` method of+ :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed+ without a deprecation period (this was needed because after the+ introduction of the+ :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and+ switching built-in spider middlewares to it those middlewares need the+ :class:`~scrapy.crawler.Crawler` instance at run time). Please use+ ``from_crawler()`` instead.+.. _release-2.12.0:Scrapy 2.12.0 (2024-11-18)diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 567a875b6..2211a822f 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -189,6 +189,19 @@ one or more of these methods::param spider: the spider which raised the exception:type spider: :class:`~scrapy.Spider` object+Base class for custom spider middlewares+----------------------------------------++Scrapy provides a base class for custom spider middlewares. It's not required+to use it but it can help with simplifying middleware implementations and+reducing the amount of boilerplate code in :ref:`universal middlewares+<universal-spider-middleware>`.++.. module:: scrapy.spidermiddlewares.base++.. autoclass:: BaseSpiderMiddleware+ :members:+.. _topics-spider-middleware-ref:Built-in spider middleware referencediff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pynew file mode 100644index 000000000..650192095--- /dev/null+++ b/scrapy/spidermiddlewares/base.py@@ -0,0 +1,97 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++from scrapy import Request, Spider++if TYPE_CHECKING:+ from collections.abc import AsyncIterable, Iterable++ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy.crawler import Crawler+ from scrapy.http import Response+++class BaseSpiderMiddleware:+ """Optional base class for spider middlewares.++ This class provides helper methods for asynchronous ``process_spider_output``+ methods. Middlewares that don't have a ``process_spider_output`` method don't need+ to use it.++ You can override the+ :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request`+ method to add processing code for requests and the+ :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item`+ method to add processing code for items. These methods take a single+ request or item from the spider output iterable and return a request or+ item (the same or a new one), or ``None`` to remove this request or item+ from the processing.+ """++ def __init__(self, crawler: Crawler):+ self.crawler: Crawler = crawler++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ return cls(crawler)++ def process_spider_output(+ self, response: Response, result: Iterable[Any], spider: Spider+ ) -> Iterable[Any]:+ for o in result:+ if isinstance(o, Request):+ o = self.get_processed_request(o, response)+ else:+ o = self.get_processed_item(o, response)+ if o is not None:+ yield o++ async def process_spider_output_async(+ self, response: Response, result: AsyncIterable[Any], spider: Spider+ ) -> AsyncIterable[Any]:+ async for o in result:+ if isinstance(o, Request):+ o = self.get_processed_request(o, response)+ else:+ o = self.get_processed_item(o, response)+ if o is not None:+ yield o++ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ """Return a processed request from the spider output.++ This method is called with a single request from the spider output.+ It should return the same or a different request, or ``None`` to+ ignore it.++ :param request: the input request+ :type request: :class:`~scrapy.Request` object++ :param response: the response being processed+ :type response: :class:`~scrapy.http.Response` object++ :return: the processed request or ``None``+ """+ return request++ def get_processed_item(self, item: Any, response: Response) -> Any:+ """Return a processed item from the spider output.++ This method is called with a single item from the spider output.+ It should return the same or a different item, or ``None`` to+ ignore it.++ :param item: the input item+ :type item: item object++ :param response: the response being processed+ :type response: :class:`~scrapy.http.Response` object++ :return: the processed item or ``None``+ """+ return itemdiff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 3164c1c03..65905f483 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -9,7 +9,7 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any-from scrapy.http import Request, Response+from scrapy.spidermiddlewares.base import BaseSpiderMiddlewareif TYPE_CHECKING:from collections.abc import AsyncIterable, Iterable@@ -19,14 +19,17 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawler+ from scrapy.http import Request, Responsefrom scrapy.statscollectors import StatsCollectorlogger = logging.getLogger(__name__)-class DepthMiddleware:- def __init__(+class DepthMiddleware(BaseSpiderMiddleware):+ crawler: Crawler++ def __init__( # pylint: disable=super-init-not-calledself,maxdepth: int,stats: StatsCollector,@@ -45,21 +48,22 @@ class DepthMiddleware:verbose = settings.getbool("DEPTH_STATS_VERBOSE")prio = settings.getint("DEPTH_PRIORITY")assert crawler.stats- return cls(maxdepth, crawler.stats, verbose, prio)+ o = cls(maxdepth, crawler.stats, verbose, prio)+ o.crawler = crawler+ return odef process_spider_output(self, response: Response, result: Iterable[Any], spider: Spider) -> Iterable[Any]:self._init_depth(response, spider)- return (r for r in result if self._filter(r, response, spider))+ yield from super().process_spider_output(response, result, spider)async def process_spider_output_async(self, response: Response, result: AsyncIterable[Any], spider: Spider) -> AsyncIterable[Any]:self._init_depth(response, spider)- async for r in result:- if self._filter(r, response, spider):- yield r+ async for o in super().process_spider_output_async(response, result, spider):+ yield odef _init_depth(self, response: Response, spider: Spider) -> None:# base case (depth=0)@@ -68,9 +72,9 @@ class DepthMiddleware:if self.verbose_stats:self.stats.inc_value("request_depth_count/0", spider=spider)- def _filter(self, request: Any, response: Response, spider: Spider) -> bool:- if not isinstance(request, Request):- return True+ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:depth = response.meta["depth"] + 1request.meta["depth"] = depthif self.prio:@@ -79,10 +83,12 @@ class DepthMiddleware:logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ",{"maxdepth": self.maxdepth, "requrl": request.url},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)- return False+ return Noneif self.verbose_stats:- self.stats.inc_value(f"request_depth_count/{depth}", spider=spider)- self.stats.max_value("request_depth_max", depth, spider=spider)- return True+ self.stats.inc_value(+ f"request_depth_count/{depth}", spider=self.crawler.spider+ )+ self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider)+ return requestdiff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 646beb911..0918c9fac 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -9,11 +9,11 @@ from __future__ import annotationsimport loggingimport reimport warnings-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKINGfrom scrapy import Spider, signalsfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.http import Request, Response+from scrapy.spidermiddlewares.base import BaseSpiderMiddlewarefrom scrapy.utils.httpobj import urlparse_cachedwarnings.warn(@@ -23,61 +23,52 @@ warnings.warn()if TYPE_CHECKING:- from collections.abc import AsyncIterable, Iterable-# typing.Self requires Python 3.11from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.http import Request, Responsefrom scrapy.statscollectors import StatsCollectorlogger = logging.getLogger(__name__)-class OffsiteMiddleware:- def __init__(self, stats: StatsCollector):+class OffsiteMiddleware(BaseSpiderMiddleware):+ crawler: Crawler++ def __init__(self, stats: StatsCollector): # pylint: disable=super-init-not-calledself.stats: StatsCollector = stats@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:assert crawler.statso = cls(crawler.stats)+ o.crawler = crawlercrawler.signals.connect(o.spider_opened, signal=signals.spider_opened)return o- def process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider- ) -> Iterable[Any]:- return (r for r in result if self._filter(r, spider))-- async def process_spider_output_async(- self, response: Response, result: AsyncIterable[Any], spider: Spider- ) -> AsyncIterable[Any]:- async for r in result:- if self._filter(r, spider):- yield r-- def _filter(self, request: Any, spider: Spider) -> bool:- if not isinstance(request, Request):- return True+ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ assert self.crawler.spiderif (request.dont_filteror request.meta.get("allow_offsite")- or self.should_follow(request, spider)+ or self.should_follow(request, self.crawler.spider)):- return True+ return requestdomain = urlparse_cached(request).hostnameif domain and domain not in self.domains_seen:self.domains_seen.add(domain)logger.debug("Filtered offsite request to %(domain)r: %(request)s",{"domain": domain, "request": request},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)- self.stats.inc_value("offsite/domains", spider=spider)- self.stats.inc_value("offsite/filtered", spider=spider)- return False+ self.stats.inc_value("offsite/domains", spider=self.crawler.spider)+ self.stats.inc_value("offsite/filtered", spider=self.crawler.spider)+ return Nonedef should_follow(self, request: Request, spider: Spider) -> bool:regex = self.host_regexdiff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex a3a1e5b92..b2ba8ba8c 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -6,7 +6,7 @@ originated it.from __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any, cast+from typing import TYPE_CHECKING, castfrom urllib.parse import urlparsefrom w3lib.url import safe_url_string@@ -14,13 +14,12 @@ from w3lib.url import safe_url_stringfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Response+from scrapy.spidermiddlewares.base import BaseSpiderMiddlewarefrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_unicodefrom scrapy.utils.url import strip_urlif TYPE_CHECKING:- from collections.abc import AsyncIterable, Iterable-# typing.Self requires Python 3.11from typing_extensions import Self@@ -327,8 +326,8 @@ def _load_policy_class(return None-class RefererMiddleware:- def __init__(self, settings: BaseSettings | None = None):+class RefererMiddleware(BaseSpiderMiddleware):+ def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-calledself.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicyif settings is not None:settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))@@ -370,23 +369,13 @@ class RefererMiddleware:cls = _load_policy_class(policy_name, warning_only=True)return cls() if cls else self.default_policy()- def process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider- ) -> Iterable[Any]:- return (self._set_referer(r, response) for r in result)-- async def process_spider_output_async(- self, response: Response, result: AsyncIterable[Any], spider: Spider- ) -> AsyncIterable[Any]:- async for r in result:- yield self._set_referer(r, response)-- def _set_referer(self, r: Any, response: Response) -> Any:- if isinstance(r, Request):- referrer = self.policy(response, r).referrer(response.url, r.url)- if referrer is not None:- r.headers.setdefault("Referer", referrer)- return r+ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ referrer = self.policy(response, request).referrer(response.url, request.url)+ if referrer is not None:+ request.headers.setdefault("Referer", referrer)+ return requestdef request_scheduled(self, request: Request, spider: Spider) -> None:# check redirected request to patch "Referer" header if necessarydiff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex a1cd1bb7c..177c19e1b 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -7,72 +7,49 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging-import warnings-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.http import Request, Response+from scrapy.exceptions import NotConfigured+from scrapy.spidermiddlewares.base import BaseSpiderMiddlewareif TYPE_CHECKING:- from collections.abc import AsyncIterable, Iterable-# typing.Self requires Python 3.11from typing_extensions import Self- from scrapy import Spiderfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettings+ from scrapy.http import Request, Responselogger = logging.getLogger(__name__)-class UrlLengthMiddleware:- def __init__(self, maxlength: int):- self.maxlength: int = maxlength+class UrlLengthMiddleware(BaseSpiderMiddleware):+ crawler: Crawler- @classmethod- def from_settings(cls, settings: BaseSettings) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings)+ def __init__(self, maxlength: int): # pylint: disable=super-init-not-called+ self.maxlength: int = maxlength@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- return cls._from_settings(crawler.settings)-- @classmethod- def _from_settings(cls, settings: BaseSettings) -> Self:- maxlength = settings.getint("URLLENGTH_LIMIT")+ maxlength = crawler.settings.getint("URLLENGTH_LIMIT")if not maxlength:raise NotConfigured- return cls(maxlength)-- def process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider- ) -> Iterable[Any]:- return (r for r in result if self._filter(r, spider))-- async def process_spider_output_async(- self, response: Response, result: AsyncIterable[Any], spider: Spider- ) -> AsyncIterable[Any]:- async for r in result:- if self._filter(r, spider):- yield r-- def _filter(self, request: Any, spider: Spider) -> bool:- if isinstance(request, Request) and len(request.url) > self.maxlength:- logger.info(- "Ignoring link (url length > %(maxlength)d): %(url)s ",- {"maxlength": self.maxlength, "url": request.url},- extra={"spider": spider},- )- assert spider.crawler.stats- spider.crawler.stats.inc_value(- "urllength/request_ignored_count", spider=spider- )- return False- return True+ o = cls(maxlength)+ o.crawler = crawler+ return o++ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ if len(request.url) <= self.maxlength:+ return request+ logger.info(+ "Ignoring link (url length > %(maxlength)d): %(url)s ",+ {"maxlength": self.maxlength, "url": request.url},+ extra={"spider": self.crawler.spider},+ )+ assert self.crawler.stats+ self.crawler.stats.inc_value(+ "urllength/request_ignored_count", spider=self.crawler.spider+ )+ return Nonediff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.pynew file mode 100644index 000000000..46be879f3--- /dev/null+++ b/tests/test_spidermiddleware_base.py@@ -0,0 +1,120 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++import pytest++from scrapy import Request, Spider+from scrapy.http import Response+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware+from scrapy.utils.test import get_crawler++if TYPE_CHECKING:+ from scrapy.crawler import Crawler+++def crawler() -> Crawler:+ return get_crawler(Spider)+++def test_trivial(crawler):+ class TrivialSpiderMiddleware(BaseSpiderMiddleware):+ pass++ mw = TrivialSpiderMiddleware.from_crawler(crawler)+ assert hasattr(mw, "crawler")+ assert mw.crawler is crawler+ test_req = Request("data:,")+ spider_output = [test_req, {"foo": "bar"}]+ processed = list(+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)+ )+ assert processed == [test_req, {"foo": "bar"}]+++def test_processed_request(crawler):+ class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):+ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ if request.url == "data:2,":+ return None+ if request.url == "data:3,":+ return Request("data:30,")+ return request++ mw = ProcessReqSpiderMiddleware.from_crawler(crawler)+ test_req1 = Request("data:1,")+ test_req2 = Request("data:2,")+ test_req3 = Request("data:3,")+ spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]+ processed = list(+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)+ )+ assert len(processed) == 3+ assert isinstance(processed[0], Request)+ assert processed[0].url == "data:1,"+ assert processed[1] == {"foo": "bar"}+ assert isinstance(processed[2], Request)+ assert processed[2].url == "data:30,"+++def test_processed_item(crawler):+ class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):+ def get_processed_item(self, item: Any, response: Response) -> Any:+ if item["foo"] == 2:+ return None+ if item["foo"] == 3:+ item["foo"] = 30+ return item++ mw = ProcessItemSpiderMiddleware.from_crawler(crawler)+ test_req = Request("data:,")+ spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]+ processed = list(+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)+ )+ assert processed == [{"foo": 1}, test_req, {"foo": 30}]+++def test_processed_both(crawler):+ class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):+ def get_processed_request(+ self, request: Request, response: Response+ ) -> Request | None:+ if request.url == "data:2,":+ return None+ if request.url == "data:3,":+ return Request("data:30,")+ return request++ def get_processed_item(self, item: Any, response: Response) -> Any:+ if item["foo"] == 2:+ return None+ if item["foo"] == 3:+ item["foo"] = 30+ return item++ mw = ProcessBothSpiderMiddleware.from_crawler(crawler)+ test_req1 = Request("data:1,")+ test_req2 = Request("data:2,")+ test_req3 = Request("data:3,")+ spider_output = [+ test_req1,+ {"foo": 1},+ {"foo": 2},+ test_req2,+ {"foo": 3},+ test_req3,+ ]+ processed = list(+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)+ )+ assert len(processed) == 4+ assert isinstance(processed[0], Request)+ assert processed[0].url == "data:1,"+ assert processed[1] == {"foo": 1}+ assert processed[2] == {"foo": 30}+ assert isinstance(processed[3], Request)+ assert processed[3].url == "data:30,"diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex dfcc141c3..9b4aa624c 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -1,19 +1,18 @@from scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.depth import DepthMiddlewarefrom scrapy.spiders import Spider-from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.test import get_crawlerclass TestDepthMiddleware:def setup_method(self):- crawler = get_crawler(Spider)+ crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})self.spider = crawler._create_spider("scrapytest.org")- self.stats = StatsCollector(crawler)+ self.stats = crawler.statsself.stats.open_spider(self.spider)- self.mw = DepthMiddleware(1, self.stats, True)+ self.mw = DepthMiddleware.from_crawler(crawler)def test_process_spider_output(self):req = Request("http://scrapytest.org")diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex f4563a0a4..e4f4b8f9b 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawlerclass TestOffsiteMiddleware:def setup_method(self):crawler = get_crawler(Spider)- self.spider = crawler._create_spider(**self._get_spiderargs())+ self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs())self.mw = OffsiteMiddleware.from_crawler(crawler)self.mw.spider_opened(self.spider)