Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Run errback output through the spider middlewares on download errors (#7992)

Adrian948e8e31dff9
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 07bc0b658..f6bdb6383 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -122,6 +122,11 @@ one or more of these methods:
This method is an :term:`asynchronous generator` called with the
results from the spider after the spider has processed the response.
+ .. versionchanged:: VERSION
+ It is also called with the output of a request :ref:`errback
+ <errbacks>`, with *response* set to ``None``, when the errback runs
+ because a download failed.
+
*result* is lazy: a generator callback runs as *result* is iterated, so
code that runs before that iteration runs before the callback body.
@@ -132,8 +137,9 @@ one or more of these methods:
.. seealso:: :ref:`universal-spider-middleware`.
:param response: the response which generated this output from the
- spider
- :type response: :class:`~scrapy.http.Response` object
+ spider, or ``None`` if the output comes from an errback called
+ because a download failed
+ :type response: :class:`~scrapy.http.Response` object or ``None``
:param result: the results from the spider
:type result: an :term:`asynchronous iterable` of
@@ -149,9 +155,14 @@ one or more of these methods:
.. method:: process_spider_exception(response, exception)
- This method is called when a spider callback or a
- :meth:`process_spider_output` method (from a previous spider
- middleware) raises an exception.
+ This method is called when a spider callback, a request :ref:`errback
+ <errbacks>` or a :meth:`process_spider_output` method (from a previous
+ spider middleware) raises an exception.
+
+ .. versionchanged:: VERSION
+ It is also called for exceptions raised while iterating the output
+ of an errback that runs because a download failed, with *response*
+ set to ``None``.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
@@ -167,8 +178,8 @@ one or more of these methods:
:meth:`process_spider_exception` will be called.
:param response: the response being processed when the exception was
- raised
- :type response: :class:`~scrapy.http.Response` object
+ raised, or ``None`` if there was no response
+ :type response: :class:`~scrapy.http.Response` object or ``None``
:param exception: the exception raised
:type exception: :exc:`Exception` object
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 02a316852..bf51c9a49 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -288,8 +288,11 @@ class Scraper:
return
try:
- # call the request errback with the downloader error
- output = await self.call_spider_async(result, request)
+ # call the request errback with the downloader error and the spider
+ # middlewares with its output
+ output = await self.spidermw._scrape_failure_async(
+ self.call_spider_async, result, request
+ )
except Exception as spider_exc:
# the errback didn't silence the exception
assert self.crawler.spider
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 6d394314b..c88566c2a 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -100,7 +100,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
async def _evaluate_iterable(
self,
- response: Response,
+ response: Response | None,
iterable: AsyncIterator[_T],
exception_processor_index: int,
recover_to: MutableAsyncChain[_T],
@@ -118,7 +118,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
def _process_spider_exception(
self,
- response: Response,
+ response: Response | None,
exception: Exception,
start_index: int = 0,
) -> MutableAsyncChain[_T]:
@@ -165,7 +165,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
def _process_spider_output(
self,
- response: Response,
+ response: Response | None,
result: AsyncIterator[_T],
start_index: int = 0,
) -> MutableAsyncChain[_T]:
@@ -186,7 +186,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
return MutableAsyncChain(result, recovered)
async def _process_callback_output(
- self, response: Response, result: AsyncIterator[_T]
+ self, response: Response | None, result: AsyncIterator[_T]
) -> MutableAsyncChain[_T]:
recovered: MutableAsyncChain[_T] = MutableAsyncChain()
result = self._evaluate_iterable(response, result, 0, recovered)
@@ -241,6 +241,20 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
await _process_pending_io()
return self._process_spider_exception(response, ex)
+ async def _scrape_failure_async(
+ self,
+ scrape_func: ScrapeFunc[_T],
+ failure: Failure,
+ request: Request,
+ ) -> MutableAsyncChain[_T]:
+ # There is no response to run the process_spider_input chain on, so the
+ # errback output enters the process_spider_output chain with None as the
+ # response. Exceptions from the errback itself are left to the caller,
+ # which tells apart the download error from a different one.
+ it: Iterable[_T] | AsyncIterator[_T] = await scrape_func(failure, request)
+ ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)
+ return await self._process_callback_output(None, ait)
+
async def process_start(
self, spider: Spider | None = None
) -> AsyncIterator[Any] | None:
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index d2e9c72ee..a8ba9192b 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -83,8 +83,9 @@ class BaseSpiderMiddleware:
:type request: :class:`~scrapy.Request` object
:param response: the response being processed
- :type response: :class:`~scrapy.http.Response` object or ``None`` for
- start requests
+ :type response: :class:`~scrapy.http.Response` object, or ``None`` for
+ start requests and for the output of a request errback called
+ because a download failed
:return: the processed request or ``None``
"""
@@ -101,8 +102,9 @@ class BaseSpiderMiddleware:
:type item: item object
:param response: the response being processed
- :type response: :class:`~scrapy.http.Response` object or ``None`` for
- start items
+ :type response: :class:`~scrapy.http.Response` object, or ``None`` for
+ start items and for the output of a request errback called because
+ a download failed
:return: the processed item or ``None``
"""
diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py
index 5d76b60d2..678815a07 100644
--- a/scrapy/spidermiddlewares/start.py
+++ b/scrapy/spidermiddlewares/start.py
@@ -1,12 +1,13 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
+
+from scrapy import Request
from .base import BaseSpiderMiddleware
if TYPE_CHECKING:
- from scrapy.http import Request
- from scrapy.http.response import Response
+ from collections.abc import AsyncIterator
class StartSpiderMiddleware(BaseSpiderMiddleware):
@@ -23,9 +24,8 @@ class StartSpiderMiddleware(BaseSpiderMiddleware):
<topics-downloader-middleware>`.
"""
- def get_processed_request(
- self, request: Request, response: Response | None
- ) -> Request | None:
- if response is None:
- request.meta.setdefault("is_start_request", True)
- return request
+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
+ async for o in start:
+ if isinstance(o, Request):
+ o.meta.setdefault("is_start_request", True)
+ yield o
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 3f06ba323..c764a4482 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -288,6 +288,53 @@ class UnhandledExceptionSpider(MockServerSpider):
raise ImportError
+# ================================================================================
+# (6) errback output after a download error
+class LogOutputMiddleware(_BaseSpiderMiddleware):
+ async def process_spider_output(self, response, result):
+ async for o in result:
+ self.crawler.spider.logger.info(
+ f"Middleware: output {o} with response {response}"
+ )
+ yield o
+
+ def process_spider_exception(self, response, exception):
+ self.crawler.spider.logger.info(
+ f"Middleware: {exception.__class__.__name__} exception caught"
+ f" with response {response}"
+ )
+ return []
+
+
+class DownloadErrorSpider(MockServerSpider):
+ name = "DownloadErrorSpider"
+ custom_settings = {
+ "SPIDER_MIDDLEWARES": {
+ LogOutputMiddleware: 10,
+ },
+ }
+
+ async def start(self):
+ assert self.mockserver
+ yield Request(self.mockserver.url("/drop?abort=1"), errback=self.errback)
+
+ def errback(self, failure):
+ yield {"from": "errback"}
+ assert self.mockserver
+ yield Request(self.mockserver.url("/status?n=200"), callback=self.parse)
+
+ def parse(self, response):
+ self.logger.info(f"is_start_request: {response.meta.get('is_start_request')}")
+
+
+class DownloadErrorFailSpider(DownloadErrorSpider):
+ name = "DownloadErrorFailSpider"
+
+ def errback(self, failure):
+ yield {"from": "errback"}
+ raise LookupError
+
+
# ================================================================================
class TestSpiderMiddleware:
@staticmethod
@@ -490,3 +537,30 @@ class TestSpiderMiddleware:
assert log5.count(f"{method}: ImportError caught") == 1
assert "Spider error processing" in log5
assert "'item_scraped_count': 1" in log5
+
+ @coroutine_test
+ async def test_download_error_errback_output(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ """
+ (6.1) The output of an errback called because a download failed goes
+ through the process_spider_output chain, with None as the response.
+ """
+ log6 = await self.crawl_log(DownloadErrorSpider, caplog, mockserver)
+ assert "Middleware: output {'from': 'errback'} with response None" in log6
+ assert "'item_scraped_count': 1" in log6
+ assert "Crawled (200)" in log6
+ assert "is_start_request: None" in log6
+
+ @coroutine_test
+ async def test_download_error_errback_exception(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ """
+ (6.2) An exception from such an errback goes through the
+ process_spider_exception chain.
+ """
+ log6 = await self.crawl_log(DownloadErrorFailSpider, caplog, mockserver)
+ assert "Middleware: output {'from': 'errback'} with response None" in log6
+ assert "Middleware: LookupError exception caught with response None" in log6
+ assert "'item_scraped_count': 1" in log6