Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove wrapping callbck exceptions into Failures. (#8363)

Andrey Rakhmatullin67bf4076c910
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index d9e053b89..279f51821 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -16,7 +16,6 @@ from scrapy.http import Request, Response
from scrapy.utils import _colorize
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import _aiter_errback, _schedule_coro, deferred_from_coro
-from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
@@ -24,8 +23,6 @@ if TYPE_CHECKING:
import argparse
from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable
- from twisted.python.failure import Failure
-
from scrapy.http.request import CallbackT
from scrapy.spiders import Spider
@@ -135,10 +132,11 @@ class Command(BaseRunSpiderCommand):
max_requests = max(self.requests)
return max(max_items, max_requests)
- def handle_exception(self, _failure: Failure) -> None:
+ @staticmethod
+ def handle_exception(exc: Exception) -> None:
logger.error(
"An error is caught while iterating the async iterable",
- exc_info=failure_to_exc_info(_failure),
+ exc_info=exc,
)
@overload
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index b192750c4..f0fb1a7b9 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -273,8 +273,8 @@ class Scraper:
output = await self.spidermw.scrape_response_async(
self.call_spider_async, result, request
)
- except Exception:
- self.handle_spider_error(Failure(), request, result)
+ except Exception as spider_exc:
+ self._handle_spider_error(spider_exc, request, result)
else:
await self.handle_spider_output_async(output, request, result)
return
@@ -309,7 +309,7 @@ class Scraper:
)
if spider_exc is not result.value:
# the errback raised a different exception, handle it
- self.handle_spider_error(Failure(), request, result)
+ self._handle_spider_error(spider_exc, request, result)
else:
await self.handle_spider_output_async(output, request, result)
@@ -385,10 +385,23 @@ class Scraper:
_failure: Failure,
request: Request,
response: Response | Failure,
+ ) -> None: # pragma: no cover
+ warnings.warn(
+ "Scraper.handle_spider_error() is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ assert _failure.value is not None
+ self._handle_spider_error(_failure.value, request, response)
+
+ def _handle_spider_error(
+ self,
+ exc: BaseException,
+ request: Request,
+ response: Response | Failure,
) -> None:
"""Handle an exception raised by a spider callback or errback."""
assert self.crawler.spider
- exc = _failure.value
if isinstance(exc, CloseSpider):
_schedule_coro(
self.crawler.engine.close_spider_async(
@@ -396,23 +409,24 @@ class Scraper:
)
)
return
+ failure = Failure(exc)
logkws = self.logformatter.spider_error(
- _failure, request, response, self.crawler.spider
+ failure, request, response, self.crawler.spider
)
logger.log(
*logformatter_adapter(logkws),
- exc_info=failure_to_exc_info(_failure),
+ exc_info=exc,
extra={"spider": self.crawler.spider},
)
self.signals.send_catch_log(
signal=signals.spider_error,
- failure=_failure,
+ failure=failure,
response=response,
spider=self.crawler.spider,
)
stats = self.crawler.stats
stats.inc_value("spider_exceptions/count")
- stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")
+ stats.inc_value(f"spider_exceptions/{exc.__class__.__name__}")
def handle_spider_output(
self,
@@ -447,7 +461,7 @@ class Scraper:
stacklevel=2,
)
result = as_async_generator(result)
- it = _aiter_errback(result, self.handle_spider_error, request, response)
+ it = _aiter_errback(result, self._handle_spider_error, request, response)
if is_asyncio_available():
await _parallel_asyncio(
it, self.concurrent_items, self._process_spidermw_output_async, response
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index dc52b06b9..ea8cd4699 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -469,20 +469,20 @@ def iter_errback(
async def _aiter_errback(
aiterable: AsyncIterator[_T],
- errback: Callable[Concatenate[Failure, _P], Any],
+ errback: Callable[Concatenate[Exception, _P], Any],
*a: _P.args,
**kw: _P.kwargs,
) -> AsyncIterator[_T]:
- """Wrap an async iterable calling an errback if an error is caught while
- iterating it."""
+ """Wrap an async iterable calling *errback* with the exception if one is
+ raised while iterating it."""
it = aiter(aiterable)
while True:
try:
yield await anext(it)
except StopAsyncIteration:
break
- except Exception:
- errback(failure.Failure(), *a, **kw)
+ except Exception as exc:
+ errback(exc, *a, **kw)
async def aiter_errback(
@@ -498,7 +498,11 @@ async def aiter_errback(
category=ScrapyDeprecationWarning,
stacklevel=2,
)
- async for item in _aiter_errback(aiterable, errback, *a, **kw):
+
+ def failure_errback(exc: Exception, /, *args: _P.args, **kwargs: _P.kwargs) -> Any:
+ return errback(failure.Failure(exc), *args, **kwargs)
+
+ async for item in _aiter_errback(aiterable, failure_errback, *a, **kw):
yield item
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 0236a36d4..d3f60fb8a 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -83,7 +83,7 @@ class TestAiterErrback:
for x in range(10):
yield x
- errors: list[Failure] = []
+ errors: list[Exception] = []
out = await collect_asyncgen(_aiter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@@ -96,11 +96,11 @@ class TestAiterErrback:
1 / 0
yield x
- errors: list[Failure] = []
+ errors: list[Exception] = []
out = await collect_asyncgen(_aiter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
- assert isinstance(errors[0].value, ZeroDivisionError)
+ assert isinstance(errors[0], ZeroDivisionError)
class TestAsyncDefTestsuite: