Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Offer an unhandled exception to each process_spider_exception method only once (#7996)

Adrian9344d60dd1ca
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 8d9aba60f..4d82c9265 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import logging
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
+from contextlib import suppress
from functools import wraps
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
@@ -108,6 +109,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
async for r in iterable:
yield r
except Exception as ex:
+ if getattr(ex, "_spidermw_unhandled", False):
+ raise
exception_result: MutableAsyncChain[_T] = self._process_spider_exception(
response, ex, exception_processor_index
)
@@ -147,6 +150,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
f"or an iterable, got {type(result)}"
)
raise _InvalidOutput(msg)
+ # Every remaining middleware declined to handle the exception, so the
+ # outer process_spider_output layers must let it through instead of
+ # offering it to those middlewares again.
+ with suppress(AttributeError):
+ exception._spidermw_unhandled = True # type: ignore[attr-defined]
raise exception
def _process_spider_output(
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 74051d1db..41a6ac9a6 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -254,6 +254,39 @@ class GeneratorOutputChainSpider(MockServerSpider):
yield {"processed": ["parse-second-item"]}
+# ================================================================================
+# (5) an exception from a spider callback (generator) that no process_spider_exception
+# method handles, with middlewares that define a process_spider_output method
+class FirstUnhandledMiddleware(_GeneratorDoNothingMiddleware):
+ pass
+
+
+class SecondUnhandledMiddleware(_GeneratorDoNothingMiddleware):
+ pass
+
+
+class ThirdUnhandledMiddleware(_GeneratorDoNothingMiddleware):
+ pass
+
+
+class UnhandledExceptionSpider(Spider):
+ name = "UnhandledExceptionSpider"
+ custom_settings = {
+ "SPIDER_MIDDLEWARES": {
+ FirstUnhandledMiddleware: 30,
+ SecondUnhandledMiddleware: 20,
+ ThirdUnhandledMiddleware: 10,
+ },
+ }
+
+ async def start(self):
+ yield Request(self.mockserver.url("/status?n=200"))
+
+ def parse(self, response):
+ yield {"processed": ["parse-first-item"]}
+ raise ImportError
+
+
# ================================================================================
class TestSpiderMiddleware:
mockserver: MockServer
@@ -437,3 +470,20 @@ class TestSpiderMiddleware:
assert str(item_from_callback) in log4
assert str(item_recovered) in log4
assert "parse-second-item" not in log4
+
+ @coroutine_test
+ async def test_unhandled_exception(self, caplog: pytest.LogCaptureFixture) -> None:
+ """
+ (5) An exception that no process_spider_exception method handles should be
+ offered to each of them only once, and then reach the spider error log.
+ """
+ log5 = await self.crawl_log(UnhandledExceptionSpider, caplog)
+ for middleware in (
+ FirstUnhandledMiddleware,
+ SecondUnhandledMiddleware,
+ ThirdUnhandledMiddleware,
+ ):
+ method = f"{middleware.__name__}.process_spider_exception"
+ assert log5.count(f"{method}: ImportError caught") == 1
+ assert "Spider error processing" in log5
+ assert "'item_scraped_count': 1" in log5