Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Base class for universal spider middlewares (#6693)

Andrey Rakhmatullindaf9db72b2dc
diff --git a/docs/news.rst b/docs/news.rst
index 9a68f8852..9f476ee21 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,23 @@
Release notes
=============
+.. _release-VERSION:
+
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The ``from_settings()`` method of
+ :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed
+ without a deprecation period (this was needed because after the
+ introduction of the
+ :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and
+ switching built-in spider middlewares to it those middlewares need the
+ :class:`~scrapy.crawler.Crawler` instance at run time). Please use
+ ``from_crawler()`` instead.
+
.. _release-2.12.0:
Scrapy 2.12.0 (2024-11-18)
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 567a875b6..2211a822f 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -189,6 +189,19 @@ one or more of these methods:
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
+Base class for custom spider middlewares
+----------------------------------------
+
+Scrapy provides a base class for custom spider middlewares. It's not required
+to use it but it can help with simplifying middleware implementations and
+reducing the amount of boilerplate code in :ref:`universal middlewares
+<universal-spider-middleware>`.
+
+.. module:: scrapy.spidermiddlewares.base
+
+.. autoclass:: BaseSpiderMiddleware
+ :members:
+
.. _topics-spider-middleware-ref:
Built-in spider middleware reference
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
new file mode 100644
index 000000000..650192095
--- /dev/null
+++ b/scrapy/spidermiddlewares/base.py
@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy import Request, Spider
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterable, Iterable
+
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
+
+class BaseSpiderMiddleware:
+ """Optional base class for spider middlewares.
+
+ This class provides helper methods for asynchronous ``process_spider_output``
+ methods. Middlewares that don't have a ``process_spider_output`` method don't need
+ to use it.
+
+ You can override the
+ :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request`
+ method to add processing code for requests and the
+ :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item`
+ method to add processing code for items. These methods take a single
+ request or item from the spider output iterable and return a request or
+ item (the same or a new one), or ``None`` to remove this request or item
+ from the processing.
+ """
+
+ def __init__(self, crawler: Crawler):
+ self.crawler: Crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ return cls(crawler)
+
+ def process_spider_output(
+ self, response: Response, result: Iterable[Any], spider: Spider
+ ) -> Iterable[Any]:
+ for o in result:
+ if isinstance(o, Request):
+ o = self.get_processed_request(o, response)
+ else:
+ o = self.get_processed_item(o, response)
+ if o is not None:
+ yield o
+
+ async def process_spider_output_async(
+ self, response: Response, result: AsyncIterable[Any], spider: Spider
+ ) -> AsyncIterable[Any]:
+ async for o in result:
+ if isinstance(o, Request):
+ o = self.get_processed_request(o, response)
+ else:
+ o = self.get_processed_item(o, response)
+ if o is not None:
+ yield o
+
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ """Return a processed request from the spider output.
+
+ This method is called with a single request from the spider output.
+ It should return the same or a different request, or ``None`` to
+ ignore it.
+
+ :param request: the input request
+ :type request: :class:`~scrapy.Request` object
+
+ :param response: the response being processed
+ :type response: :class:`~scrapy.http.Response` object
+
+ :return: the processed request or ``None``
+ """
+ return request
+
+ def get_processed_item(self, item: Any, response: Response) -> Any:
+ """Return a processed item from the spider output.
+
+ This method is called with a single item from the spider output.
+ It should return the same or a different item, or ``None`` to
+ ignore it.
+
+ :param item: the input item
+ :type item: item object
+
+ :param response: the response being processed
+ :type response: :class:`~scrapy.http.Response` object
+
+ :return: the processed item or ``None``
+ """
+ return item
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 3164c1c03..65905f483 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -9,7 +9,7 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
-from scrapy.http import Request, Response
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
if TYPE_CHECKING:
from collections.abc import AsyncIterable, Iterable
@@ -19,14 +19,17 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
+ from scrapy.http import Request, Response
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
-class DepthMiddleware:
- def __init__(
+class DepthMiddleware(BaseSpiderMiddleware):
+ crawler: Crawler
+
+ def __init__( # pylint: disable=super-init-not-called
self,
maxdepth: int,
stats: StatsCollector,
@@ -45,21 +48,22 @@ class DepthMiddleware:
verbose = settings.getbool("DEPTH_STATS_VERBOSE")
prio = settings.getint("DEPTH_PRIORITY")
assert crawler.stats
- return cls(maxdepth, crawler.stats, verbose, prio)
+ o = cls(maxdepth, crawler.stats, verbose, prio)
+ o.crawler = crawler
+ return o
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
self._init_depth(response, spider)
- return (r for r in result if self._filter(r, response, spider))
+ yield from super().process_spider_output(response, result, spider)
async def process_spider_output_async(
self, response: Response, result: AsyncIterable[Any], spider: Spider
) -> AsyncIterable[Any]:
self._init_depth(response, spider)
- async for r in result:
- if self._filter(r, response, spider):
- yield r
+ async for o in super().process_spider_output_async(response, result, spider):
+ yield o
def _init_depth(self, response: Response, spider: Spider) -> None:
# base case (depth=0)
@@ -68,9 +72,9 @@ class DepthMiddleware:
if self.verbose_stats:
self.stats.inc_value("request_depth_count/0", spider=spider)
- def _filter(self, request: Any, response: Response, spider: Spider) -> bool:
- if not isinstance(request, Request):
- return True
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
depth = response.meta["depth"] + 1
request.meta["depth"] = depth
if self.prio:
@@ -79,10 +83,12 @@ class DepthMiddleware:
logger.debug(
"Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
{"maxdepth": self.maxdepth, "requrl": request.url},
- extra={"spider": spider},
+ extra={"spider": self.crawler.spider},
)
- return False
+ return None
if self.verbose_stats:
- self.stats.inc_value(f"request_depth_count/{depth}", spider=spider)
- self.stats.max_value("request_depth_max", depth, spider=spider)
- return True
+ self.stats.inc_value(
+ f"request_depth_count/{depth}", spider=self.crawler.spider
+ )
+ self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider)
+ return request
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 646beb911..0918c9fac 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -9,11 +9,11 @@ from __future__ import annotations
import logging
import re
import warnings
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.http import Request, Response
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.httpobj import urlparse_cached
warnings.warn(
@@ -23,61 +23,52 @@ warnings.warn(
)
if TYPE_CHECKING:
- from collections.abc import AsyncIterable, Iterable
-
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.http import Request, Response
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
-class OffsiteMiddleware:
- def __init__(self, stats: StatsCollector):
+class OffsiteMiddleware(BaseSpiderMiddleware):
+ crawler: Crawler
+
+ def __init__(self, stats: StatsCollector): # pylint: disable=super-init-not-called
self.stats: StatsCollector = stats
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
+ o.crawler = crawler
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
- def process_spider_output(
- self, response: Response, result: Iterable[Any], spider: Spider
- ) -> Iterable[Any]:
- return (r for r in result if self._filter(r, spider))
-
- async def process_spider_output_async(
- self, response: Response, result: AsyncIterable[Any], spider: Spider
- ) -> AsyncIterable[Any]:
- async for r in result:
- if self._filter(r, spider):
- yield r
-
- def _filter(self, request: Any, spider: Spider) -> bool:
- if not isinstance(request, Request):
- return True
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ assert self.crawler.spider
if (
request.dont_filter
or request.meta.get("allow_offsite")
- or self.should_follow(request, spider)
+ or self.should_follow(request, self.crawler.spider)
):
- return True
+ return request
domain = urlparse_cached(request).hostname
if domain and domain not in self.domains_seen:
self.domains_seen.add(domain)
logger.debug(
"Filtered offsite request to %(domain)r: %(request)s",
{"domain": domain, "request": request},
- extra={"spider": spider},
+ extra={"spider": self.crawler.spider},
)
- self.stats.inc_value("offsite/domains", spider=spider)
- self.stats.inc_value("offsite/filtered", spider=spider)
- return False
+ self.stats.inc_value("offsite/domains", spider=self.crawler.spider)
+ self.stats.inc_value("offsite/filtered", spider=self.crawler.spider)
+ return None
def should_follow(self, request: Request, spider: Spider) -> bool:
regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index a3a1e5b92..b2ba8ba8c 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -6,7 +6,7 @@ originated it.
from __future__ import annotations
import warnings
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, cast
from urllib.parse import urlparse
from w3lib.url import safe_url_string
@@ -14,13 +14,12 @@ from w3lib.url import safe_url_string
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
if TYPE_CHECKING:
- from collections.abc import AsyncIterable, Iterable
-
# typing.Self requires Python 3.11
from typing_extensions import Self
@@ -327,8 +326,8 @@ def _load_policy_class(
return None
-class RefererMiddleware:
- def __init__(self, settings: BaseSettings | None = None):
+class RefererMiddleware(BaseSpiderMiddleware):
+ def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called
self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy
if settings is not None:
settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
@@ -370,23 +369,13 @@ class RefererMiddleware:
cls = _load_policy_class(policy_name, warning_only=True)
return cls() if cls else self.default_policy()
- def process_spider_output(
- self, response: Response, result: Iterable[Any], spider: Spider
- ) -> Iterable[Any]:
- return (self._set_referer(r, response) for r in result)
-
- async def process_spider_output_async(
- self, response: Response, result: AsyncIterable[Any], spider: Spider
- ) -> AsyncIterable[Any]:
- async for r in result:
- yield self._set_referer(r, response)
-
- def _set_referer(self, r: Any, response: Response) -> Any:
- if isinstance(r, Request):
- referrer = self.policy(response, r).referrer(response.url, r.url)
- if referrer is not None:
- r.headers.setdefault("Referer", referrer)
- return r
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ referrer = self.policy(response, request).referrer(response.url, request.url)
+ if referrer is not None:
+ request.headers.setdefault("Referer", referrer)
+ return request
def request_scheduled(self, request: Request, spider: Spider) -> None:
# check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index a1cd1bb7c..177c19e1b 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -7,72 +7,49 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
-import warnings
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Request, Response
+from scrapy.exceptions import NotConfigured
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
if TYPE_CHECKING:
- from collections.abc import AsyncIterable, Iterable
-
# typing.Self requires Python 3.11
from typing_extensions import Self
- from scrapy import Spider
from scrapy.crawler import Crawler
- from scrapy.settings import BaseSettings
+ from scrapy.http import Request, Response
logger = logging.getLogger(__name__)
-class UrlLengthMiddleware:
- def __init__(self, maxlength: int):
- self.maxlength: int = maxlength
+class UrlLengthMiddleware(BaseSpiderMiddleware):
+ crawler: Crawler
- @classmethod
- def from_settings(cls, settings: BaseSettings) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings)
+ def __init__(self, maxlength: int): # pylint: disable=super-init-not-called
+ self.maxlength: int = maxlength
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- return cls._from_settings(crawler.settings)
-
- @classmethod
- def _from_settings(cls, settings: BaseSettings) -> Self:
- maxlength = settings.getint("URLLENGTH_LIMIT")
+ maxlength = crawler.settings.getint("URLLENGTH_LIMIT")
if not maxlength:
raise NotConfigured
- return cls(maxlength)
-
- def process_spider_output(
- self, response: Response, result: Iterable[Any], spider: Spider
- ) -> Iterable[Any]:
- return (r for r in result if self._filter(r, spider))
-
- async def process_spider_output_async(
- self, response: Response, result: AsyncIterable[Any], spider: Spider
- ) -> AsyncIterable[Any]:
- async for r in result:
- if self._filter(r, spider):
- yield r
-
- def _filter(self, request: Any, spider: Spider) -> bool:
- if isinstance(request, Request) and len(request.url) > self.maxlength:
- logger.info(
- "Ignoring link (url length > %(maxlength)d): %(url)s ",
- {"maxlength": self.maxlength, "url": request.url},
- extra={"spider": spider},
- )
- assert spider.crawler.stats
- spider.crawler.stats.inc_value(
- "urllength/request_ignored_count", spider=spider
- )
- return False
- return True
+ o = cls(maxlength)
+ o.crawler = crawler
+ return o
+
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ if len(request.url) <= self.maxlength:
+ return request
+ logger.info(
+ "Ignoring link (url length > %(maxlength)d): %(url)s ",
+ {"maxlength": self.maxlength, "url": request.url},
+ extra={"spider": self.crawler.spider},
+ )
+ assert self.crawler.stats
+ self.crawler.stats.inc_value(
+ "urllength/request_ignored_count", spider=self.crawler.spider
+ )
+ return None
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
new file mode 100644
index 000000000..46be879f3
--- /dev/null
+++ b/tests/test_spidermiddleware_base.py
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy import Request, Spider
+from scrapy.http import Response
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
+
+def crawler() -> Crawler:
+ return get_crawler(Spider)
+
+
+def test_trivial(crawler):
+ class TrivialSpiderMiddleware(BaseSpiderMiddleware):
+ pass
+
+ mw = TrivialSpiderMiddleware.from_crawler(crawler)
+ assert hasattr(mw, "crawler")
+ assert mw.crawler is crawler
+ test_req = Request("data:,")
+ spider_output = [test_req, {"foo": "bar"}]
+ processed = list(
+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
+ )
+ assert processed == [test_req, {"foo": "bar"}]
+
+
+def test_processed_request(crawler):
+ class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ if request.url == "data:2,":
+ return None
+ if request.url == "data:3,":
+ return Request("data:30,")
+ return request
+
+ mw = ProcessReqSpiderMiddleware.from_crawler(crawler)
+ test_req1 = Request("data:1,")
+ test_req2 = Request("data:2,")
+ test_req3 = Request("data:3,")
+ spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
+ processed = list(
+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
+ )
+ assert len(processed) == 3
+ assert isinstance(processed[0], Request)
+ assert processed[0].url == "data:1,"
+ assert processed[1] == {"foo": "bar"}
+ assert isinstance(processed[2], Request)
+ assert processed[2].url == "data:30,"
+
+
+def test_processed_item(crawler):
+ class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
+ def get_processed_item(self, item: Any, response: Response) -> Any:
+ if item["foo"] == 2:
+ return None
+ if item["foo"] == 3:
+ item["foo"] = 30
+ return item
+
+ mw = ProcessItemSpiderMiddleware.from_crawler(crawler)
+ test_req = Request("data:,")
+ spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
+ processed = list(
+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
+ )
+ assert processed == [{"foo": 1}, test_req, {"foo": 30}]
+
+
+def test_processed_both(crawler):
+ class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
+ def get_processed_request(
+ self, request: Request, response: Response
+ ) -> Request | None:
+ if request.url == "data:2,":
+ return None
+ if request.url == "data:3,":
+ return Request("data:30,")
+ return request
+
+ def get_processed_item(self, item: Any, response: Response) -> Any:
+ if item["foo"] == 2:
+ return None
+ if item["foo"] == 3:
+ item["foo"] = 30
+ return item
+
+ mw = ProcessBothSpiderMiddleware.from_crawler(crawler)
+ test_req1 = Request("data:1,")
+ test_req2 = Request("data:2,")
+ test_req3 = Request("data:3,")
+ spider_output = [
+ test_req1,
+ {"foo": 1},
+ {"foo": 2},
+ test_req2,
+ {"foo": 3},
+ test_req3,
+ ]
+ processed = list(
+ mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
+ )
+ assert len(processed) == 4
+ assert isinstance(processed[0], Request)
+ assert processed[0].url == "data:1,"
+ assert processed[1] == {"foo": 1}
+ assert processed[2] == {"foo": 30}
+ assert isinstance(processed[3], Request)
+ assert processed[3].url == "data:30,"
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index dfcc141c3..9b4aa624c 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -1,19 +1,18 @@
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
class TestDepthMiddleware:
def setup_method(self):
- crawler = get_crawler(Spider)
+ crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
self.spider = crawler._create_spider("scrapytest.org")
- self.stats = StatsCollector(crawler)
+ self.stats = crawler.stats
self.stats.open_spider(self.spider)
- self.mw = DepthMiddleware(1, self.stats, True)
+ self.mw = DepthMiddleware.from_crawler(crawler)
def test_process_spider_output(self):
req = Request("http://scrapytest.org")
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index f4563a0a4..e4f4b8f9b 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler
class TestOffsiteMiddleware:
def setup_method(self):
crawler = get_crawler(Spider)
- self.spider = crawler._create_spider(**self._get_spiderargs())
+ self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs())
self.mw = OffsiteMiddleware.from_crawler(crawler)
self.mw.spider_opened(self.spider)