Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove wrapping callbck exceptions into Failures. (#8363)
Andrey Rakhmatullin67bf4076c910
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex d9e053b89..279f51821 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -16,7 +16,6 @@ from scrapy.http import Request, Responsefrom scrapy.utils import _colorizefrom scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.defer import _aiter_errback, _schedule_coro, deferred_from_coro-from scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.spider import spidercls_for_request@@ -24,8 +23,6 @@ if TYPE_CHECKING:import argparsefrom collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable- from twisted.python.failure import Failure-from scrapy.http.request import CallbackTfrom scrapy.spiders import Spider@@ -135,10 +132,11 @@ class Command(BaseRunSpiderCommand):max_requests = max(self.requests)return max(max_items, max_requests)- def handle_exception(self, _failure: Failure) -> None:+ @staticmethod+ def handle_exception(exc: Exception) -> None:logger.error("An error is caught while iterating the async iterable",- exc_info=failure_to_exc_info(_failure),+ exc_info=exc,)@overloaddiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex b192750c4..f0fb1a7b9 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -273,8 +273,8 @@ class Scraper:output = await self.spidermw.scrape_response_async(self.call_spider_async, result, request)- except Exception:- self.handle_spider_error(Failure(), request, result)+ except Exception as spider_exc:+ self._handle_spider_error(spider_exc, request, result)else:await self.handle_spider_output_async(output, request, result)return@@ -309,7 +309,7 @@ class Scraper:)if spider_exc is not result.value:# the errback raised a different exception, handle it- self.handle_spider_error(Failure(), request, result)+ self._handle_spider_error(spider_exc, request, result)else:await self.handle_spider_output_async(output, request, result)@@ -385,10 +385,23 @@ class Scraper:_failure: Failure,request: Request,response: Response | Failure,+ ) -> None: # pragma: no cover+ warnings.warn(+ "Scraper.handle_spider_error() is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ assert _failure.value is not None+ self._handle_spider_error(_failure.value, request, response)++ def _handle_spider_error(+ self,+ exc: BaseException,+ request: Request,+ response: Response | Failure,) -> None:"""Handle an exception raised by a spider callback or errback."""assert self.crawler.spider- exc = _failure.valueif isinstance(exc, CloseSpider):_schedule_coro(self.crawler.engine.close_spider_async(@@ -396,23 +409,24 @@ class Scraper:))return+ failure = Failure(exc)logkws = self.logformatter.spider_error(- _failure, request, response, self.crawler.spider+ failure, request, response, self.crawler.spider)logger.log(*logformatter_adapter(logkws),- exc_info=failure_to_exc_info(_failure),+ exc_info=exc,extra={"spider": self.crawler.spider},)self.signals.send_catch_log(signal=signals.spider_error,- failure=_failure,+ failure=failure,response=response,spider=self.crawler.spider,)stats = self.crawler.statsstats.inc_value("spider_exceptions/count")- stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")+ stats.inc_value(f"spider_exceptions/{exc.__class__.__name__}")def handle_spider_output(self,@@ -447,7 +461,7 @@ class Scraper:stacklevel=2,)result = as_async_generator(result)- it = _aiter_errback(result, self.handle_spider_error, request, response)+ it = _aiter_errback(result, self._handle_spider_error, request, response)if is_asyncio_available():await _parallel_asyncio(it, self.concurrent_items, self._process_spidermw_output_async, responsediff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex dc52b06b9..ea8cd4699 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -469,20 +469,20 @@ def iter_errback(async def _aiter_errback(aiterable: AsyncIterator[_T],- errback: Callable[Concatenate[Failure, _P], Any],+ errback: Callable[Concatenate[Exception, _P], Any],*a: _P.args,**kw: _P.kwargs,) -> AsyncIterator[_T]:- """Wrap an async iterable calling an errback if an error is caught while- iterating it."""+ """Wrap an async iterable calling *errback* with the exception if one is+ raised while iterating it."""it = aiter(aiterable)while True:try:yield await anext(it)except StopAsyncIteration:break- except Exception:- errback(failure.Failure(), *a, **kw)+ except Exception as exc:+ errback(exc, *a, **kw)async def aiter_errback(@@ -498,7 +498,11 @@ async def aiter_errback(category=ScrapyDeprecationWarning,stacklevel=2,)- async for item in _aiter_errback(aiterable, errback, *a, **kw):++ def failure_errback(exc: Exception, /, *args: _P.args, **kwargs: _P.kwargs) -> Any:+ return errback(failure.Failure(exc), *args, **kwargs)++ async for item in _aiter_errback(aiterable, failure_errback, *a, **kw):yield itemdiff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 0236a36d4..d3f60fb8a 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -83,7 +83,7 @@ class TestAiterErrback:for x in range(10):yield x- errors: list[Failure] = []+ errors: list[Exception] = []out = await collect_asyncgen(_aiter_errback(itergood(), errors.append))assert out == list(range(10))assert not errors@@ -96,11 +96,11 @@ class TestAiterErrback:1 / 0yield x- errors: list[Failure] = []+ errors: list[Exception] = []out = await collect_asyncgen(_aiter_errback(iterbad(), errors.append))assert out == [0, 1, 2, 3, 4]assert len(errors) == 1- assert isinstance(errors[0].value, ZeroDivisionError)+ assert isinstance(errors[0], ZeroDivisionError)class TestAsyncDefTestsuite: