Scrapy, a fast high-level web crawling & scraping framework for Python.
Run errback output through the spider middlewares on download errors (#7992)
Adrian948e8e31dff9
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 07bc0b658..f6bdb6383 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -122,6 +122,11 @@ one or more of these methods:This method is an :term:`asynchronous generator` called with theresults from the spider after the spider has processed the response.+ .. versionchanged:: VERSION+ It is also called with the output of a request :ref:`errback+ <errbacks>`, with *response* set to ``None``, when the errback runs+ because a download failed.+*result* is lazy: a generator callback runs as *result* is iterated, socode that runs before that iteration runs before the callback body.@@ -132,8 +137,9 @@ one or more of these methods:.. seealso:: :ref:`universal-spider-middleware`.:param response: the response which generated this output from the- spider- :type response: :class:`~scrapy.http.Response` object+ spider, or ``None`` if the output comes from an errback called+ because a download failed+ :type response: :class:`~scrapy.http.Response` object or ``None``:param result: the results from the spider:type result: an :term:`asynchronous iterable` of@@ -149,9 +155,14 @@ one or more of these methods:.. method:: process_spider_exception(response, exception)- This method is called when a spider callback or a- :meth:`process_spider_output` method (from a previous spider- middleware) raises an exception.+ This method is called when a spider callback, a request :ref:`errback+ <errbacks>` or a :meth:`process_spider_output` method (from a previous+ spider middleware) raises an exception.++ .. versionchanged:: VERSION+ It is also called for exceptions raised while iterating the output+ of an errback that runs because a download failed, with *response*+ set to ``None``.:meth:`process_spider_exception` should return either ``None`` or aniterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`@@ -167,8 +178,8 @@ one or more of these methods::meth:`process_spider_exception` will be called.:param response: the response being processed when the exception was- raised- :type response: :class:`~scrapy.http.Response` object+ raised, or ``None`` if there was no response+ :type response: :class:`~scrapy.http.Response` object or ``None``:param exception: the exception raised:type exception: :exc:`Exception` objectdiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 02a316852..bf51c9a49 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -288,8 +288,11 @@ class Scraper:returntry:- # call the request errback with the downloader error- output = await self.call_spider_async(result, request)+ # call the request errback with the downloader error and the spider+ # middlewares with its output+ output = await self.spidermw._scrape_failure_async(+ self.call_spider_async, result, request+ )except Exception as spider_exc:# the errback didn't silence the exceptionassert self.crawler.spiderdiff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 6d394314b..c88566c2a 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -100,7 +100,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-measync def _evaluate_iterable(self,- response: Response,+ response: Response | None,iterable: AsyncIterator[_T],exception_processor_index: int,recover_to: MutableAsyncChain[_T],@@ -118,7 +118,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-medef _process_spider_exception(self,- response: Response,+ response: Response | None,exception: Exception,start_index: int = 0,) -> MutableAsyncChain[_T]:@@ -165,7 +165,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-medef _process_spider_output(self,- response: Response,+ response: Response | None,result: AsyncIterator[_T],start_index: int = 0,) -> MutableAsyncChain[_T]:@@ -186,7 +186,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-mereturn MutableAsyncChain(result, recovered)async def _process_callback_output(- self, response: Response, result: AsyncIterator[_T]+ self, response: Response | None, result: AsyncIterator[_T]) -> MutableAsyncChain[_T]:recovered: MutableAsyncChain[_T] = MutableAsyncChain()result = self._evaluate_iterable(response, result, 0, recovered)@@ -241,6 +241,20 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meawait _process_pending_io()return self._process_spider_exception(response, ex)+ async def _scrape_failure_async(+ self,+ scrape_func: ScrapeFunc[_T],+ failure: Failure,+ request: Request,+ ) -> MutableAsyncChain[_T]:+ # There is no response to run the process_spider_input chain on, so the+ # errback output enters the process_spider_output chain with None as the+ # response. Exceptions from the errback itself are left to the caller,+ # which tells apart the download error from a different one.+ it: Iterable[_T] | AsyncIterator[_T] = await scrape_func(failure, request)+ ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)+ return await self._process_callback_output(None, ait)+async def process_start(self, spider: Spider | None = None) -> AsyncIterator[Any] | None:diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex d2e9c72ee..a8ba9192b 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -83,8 +83,9 @@ class BaseSpiderMiddleware::type request: :class:`~scrapy.Request` object:param response: the response being processed- :type response: :class:`~scrapy.http.Response` object or ``None`` for- start requests+ :type response: :class:`~scrapy.http.Response` object, or ``None`` for+ start requests and for the output of a request errback called+ because a download failed:return: the processed request or ``None``"""@@ -101,8 +102,9 @@ class BaseSpiderMiddleware::type item: item object:param response: the response being processed- :type response: :class:`~scrapy.http.Response` object or ``None`` for- start items+ :type response: :class:`~scrapy.http.Response` object, or ``None`` for+ start items and for the output of a request errback called because+ a download failed:return: the processed item or ``None``"""diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.pyindex 5d76b60d2..678815a07 100644--- a/scrapy/spidermiddlewares/start.py+++ b/scrapy/spidermiddlewares/start.py@@ -1,12 +1,13 @@from __future__ import annotations-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Any++from scrapy import Requestfrom .base import BaseSpiderMiddlewareif TYPE_CHECKING:- from scrapy.http import Request- from scrapy.http.response import Response+ from collections.abc import AsyncIteratorclass StartSpiderMiddleware(BaseSpiderMiddleware):@@ -23,9 +24,8 @@ class StartSpiderMiddleware(BaseSpiderMiddleware):<topics-downloader-middleware>`."""- def get_processed_request(- self, request: Request, response: Response | None- ) -> Request | None:- if response is None:- request.meta.setdefault("is_start_request", True)- return request+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:+ async for o in start:+ if isinstance(o, Request):+ o.meta.setdefault("is_start_request", True)+ yield odiff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 3f06ba323..c764a4482 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -288,6 +288,53 @@ class UnhandledExceptionSpider(MockServerSpider):raise ImportError+# ================================================================================+# (6) errback output after a download error+class LogOutputMiddleware(_BaseSpiderMiddleware):+ async def process_spider_output(self, response, result):+ async for o in result:+ self.crawler.spider.logger.info(+ f"Middleware: output {o} with response {response}"+ )+ yield o++ def process_spider_exception(self, response, exception):+ self.crawler.spider.logger.info(+ f"Middleware: {exception.__class__.__name__} exception caught"+ f" with response {response}"+ )+ return []+++class DownloadErrorSpider(MockServerSpider):+ name = "DownloadErrorSpider"+ custom_settings = {+ "SPIDER_MIDDLEWARES": {+ LogOutputMiddleware: 10,+ },+ }++ async def start(self):+ assert self.mockserver+ yield Request(self.mockserver.url("/drop?abort=1"), errback=self.errback)++ def errback(self, failure):+ yield {"from": "errback"}+ assert self.mockserver+ yield Request(self.mockserver.url("/status?n=200"), callback=self.parse)++ def parse(self, response):+ self.logger.info(f"is_start_request: {response.meta.get('is_start_request')}")+++class DownloadErrorFailSpider(DownloadErrorSpider):+ name = "DownloadErrorFailSpider"++ def errback(self, failure):+ yield {"from": "errback"}+ raise LookupError++# ================================================================================class TestSpiderMiddleware:@staticmethod@@ -490,3 +537,30 @@ class TestSpiderMiddleware:assert log5.count(f"{method}: ImportError caught") == 1assert "Spider error processing" in log5assert "'item_scraped_count': 1" in log5++ @coroutine_test+ async def test_download_error_errback_output(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ """+ (6.1) The output of an errback called because a download failed goes+ through the process_spider_output chain, with None as the response.+ """+ log6 = await self.crawl_log(DownloadErrorSpider, caplog, mockserver)+ assert "Middleware: output {'from': 'errback'} with response None" in log6+ assert "'item_scraped_count': 1" in log6+ assert "Crawled (200)" in log6+ assert "is_start_request: None" in log6++ @coroutine_test+ async def test_download_error_errback_exception(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ """+ (6.2) An exception from such an errback goes through the+ process_spider_exception chain.+ """+ log6 = await self.crawl_log(DownloadErrorFailSpider, caplog, mockserver)+ assert "Middleware: output {'from': 'errback'} with response None" in log6+ assert "Middleware: LookupError exception caught with response None" in log6+ assert "'item_scraped_count': 1" in log6