Scrapy, a fast high-level web crawling & scraping framework for Python.
Offer an unhandled exception to each process_spider_exception method only once (#7996)
Adrian9344d60dd1ca
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 8d9aba60f..4d82c9265 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport loggingfrom collections.abc import AsyncIterator, Callable, Coroutine, Iterable+from contextlib import suppressfrom functools import wrapsfrom inspect import isasyncgenfunction, iscoroutinefrom itertools import islice@@ -108,6 +109,8 @@ class SpiderMiddlewareManager(MiddlewareManager):async for r in iterable:yield rexcept Exception as ex:+ if getattr(ex, "_spidermw_unhandled", False):+ raiseexception_result: MutableAsyncChain[_T] = self._process_spider_exception(response, ex, exception_processor_index)@@ -147,6 +150,11 @@ class SpiderMiddlewareManager(MiddlewareManager):f"or an iterable, got {type(result)}")raise _InvalidOutput(msg)+ # Every remaining middleware declined to handle the exception, so the+ # outer process_spider_output layers must let it through instead of+ # offering it to those middlewares again.+ with suppress(AttributeError):+ exception._spidermw_unhandled = True # type: ignore[attr-defined]raise exceptiondef _process_spider_output(diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 74051d1db..41a6ac9a6 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -254,6 +254,39 @@ class GeneratorOutputChainSpider(MockServerSpider):yield {"processed": ["parse-second-item"]}+# ================================================================================+# (5) an exception from a spider callback (generator) that no process_spider_exception+# method handles, with middlewares that define a process_spider_output method+class FirstUnhandledMiddleware(_GeneratorDoNothingMiddleware):+ pass+++class SecondUnhandledMiddleware(_GeneratorDoNothingMiddleware):+ pass+++class ThirdUnhandledMiddleware(_GeneratorDoNothingMiddleware):+ pass+++class UnhandledExceptionSpider(Spider):+ name = "UnhandledExceptionSpider"+ custom_settings = {+ "SPIDER_MIDDLEWARES": {+ FirstUnhandledMiddleware: 30,+ SecondUnhandledMiddleware: 20,+ ThirdUnhandledMiddleware: 10,+ },+ }++ async def start(self):+ yield Request(self.mockserver.url("/status?n=200"))++ def parse(self, response):+ yield {"processed": ["parse-first-item"]}+ raise ImportError++# ================================================================================class TestSpiderMiddleware:mockserver: MockServer@@ -437,3 +470,20 @@ class TestSpiderMiddleware:assert str(item_from_callback) in log4assert str(item_recovered) in log4assert "parse-second-item" not in log4++ @coroutine_test+ async def test_unhandled_exception(self, caplog: pytest.LogCaptureFixture) -> None:+ """+ (5) An exception that no process_spider_exception method handles should be+ offered to each of them only once, and then reach the spider error log.+ """+ log5 = await self.crawl_log(UnhandledExceptionSpider, caplog)+ for middleware in (+ FirstUnhandledMiddleware,+ SecondUnhandledMiddleware,+ ThirdUnhandledMiddleware,+ ):+ method = f"{middleware.__name__}.process_spider_exception"+ assert log5.count(f"{method}: ImportError caught") == 1+ assert "Spider error processing" in log5+ assert "'item_scraped_count': 1" in log5