Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle exceptions and cancellation in parallel helpers. (#8318)

Andrey Rakhmatullin2aa46683b396
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index 7116b449f..7fd4f7531 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -117,27 +117,25 @@ async def _parallel_asyncio(
Tasks are created on demand, one per item, so that a *count* much larger
than the amount of work does not cost anything.
- This function is only used in
- :meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so it
- assumes that neither *callable* nor iterating *iterable* will raise an
- exception.
+ Exceptions raised by *callable_* are logged. If this coroutine is
+ cancelled, or iterating *iterable* raises an exception, the calls in
+ progress are cancelled; an exception from *iterable* is raised in an
+ :exc:`ExceptionGroup`.
"""
semaphore = asyncio.Semaphore(count)
- tasks: set[asyncio.Task[None]] = set()
async def work(item: _T) -> None:
try:
await callable_(item, *args, **kwargs)
+ except Exception:
+ logger.exception("Error processing an item")
finally:
semaphore.release()
- async for item in as_async_generator(iterable):
- await semaphore.acquire()
- task = asyncio.create_task(work(item))
- tasks.add(task)
- task.add_done_callback(tasks.discard)
- if tasks:
- await asyncio.wait(tasks)
+ async with asyncio.TaskGroup() as tg:
+ async for item in as_async_generator(iterable):
+ await semaphore.acquire()
+ tg.create_task(work(item))
class AsyncioLoopingCall(Generic[_P, _T]):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d3fa4d214..4ce1a78fb 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -6,6 +6,7 @@ from __future__ import annotations
import asyncio
import inspect
+import logging
import sys
import warnings
from asyncio import Future
@@ -48,6 +49,8 @@ if TYPE_CHECKING:
request: Request
+logger = logging.getLogger(__name__)
+
_T = TypeVar("_T")
_T2 = TypeVar("_T2")
_P = ParamSpec("_P")
@@ -212,8 +215,20 @@ class _ParallelTasks:
self._running += 1
self._coop.coiterate(self._work).addBoth(self._task_done)
- def _task_done(self, _: Any) -> None:
+ def _task_done(self, result: Any) -> None:
self._running -= 1
+ if isinstance(result, failure.Failure):
+ # circular import
+ from scrapy.utils.log import failure_to_exc_info # noqa: PLC0415
+
+ logger.error(
+ "Error processing an item",
+ exc_info=failure_to_exc_info(result),
+ )
+ # A failed task stops consuming work, which may not be exhausted
+ # yet, so another one takes over.
+ self.start()
+ return
# Only a running task can start another one, so once none is left no
# more work can come.
if not self._running:
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index ed15bceae..4fd53bd8c 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -130,6 +130,75 @@ class TestParallelAsyncio:
assert results == [0, 1, 2]
assert max(task_counts) < 100
+ @pytest.mark.parametrize("count", [1, 2])
+ @coroutine_test
+ async def test_callable_error(
+ self, count: int, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ results: list[int] = []
+
+ async def callable_(o: int) -> None:
+ if o == 1:
+ raise ValueError("Bad item")
+ results.append(o)
+
+ await _parallel_asyncio(range(4), count, callable_)
+ assert sorted(results) == [0, 2, 3]
+ assert "Error processing an item" in caplog.text
+ assert "Bad item" in caplog.text
+
+ @coroutine_test
+ async def test_cancelled(self) -> None:
+ """Cancelling the coroutine cancels the calls in progress."""
+ started: list[int] = []
+ cancelled: list[int] = []
+ all_started = asyncio.Event()
+
+ async def callable_(o: int) -> None:
+ started.append(o)
+ if len(started) == 3:
+ all_started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled.append(o)
+ raise
+
+ task = asyncio.create_task(_parallel_asyncio(range(3), 3, callable_))
+ await all_started.wait()
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert sorted(cancelled) == [0, 1, 2]
+
+ @coroutine_test
+ async def test_iteration_error(self) -> None:
+ """An exception raised while iterating is propagated in an exception
+ group, and the calls in progress are cancelled."""
+ started: list[int] = []
+ cancelled: list[int] = []
+ all_started = asyncio.Event()
+
+ async def iterable() -> AsyncGenerator[int, None]:
+ yield 0
+ yield 1
+ await all_started.wait()
+ raise ValueError("Bad iterable")
+
+ async def callable_(o: int) -> None:
+ started.append(o)
+ if len(started) == 2:
+ all_started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled.append(o)
+ raise
+
+ with pytest.RaisesGroup(pytest.RaisesExc(ValueError, match="Bad iterable")):
+ await _parallel_asyncio(iterable(), 3, callable_)
+ assert sorted(cancelled) == [0, 1]
+
_real_sleep = asyncio.sleep
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 5a277d08a..b10ddd25b 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Self
from unittest import mock
import pytest
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred, fail, inlineCallbacks
from twisted.internet.interfaces import IReadDescriptor
from twisted.internet.task import Cooperator
from zope.interface import implementer
@@ -309,6 +309,24 @@ class TestParallelAsync:
# One task per item, plus the one that finds no more work.
assert coiterate.call_count <= 4
+ @pytest.mark.parametrize("count", [1, 2])
+ @inline_callbacks_test
+ def test_callable_error(
+ self, count: int, caplog: pytest.LogCaptureFixture
+ ) -> Generator[Deferred[Any], Any, None]:
+ results: list[int] = []
+
+ def callable_(o: int) -> Deferred[None] | None:
+ if o == 1:
+ return fail(ValueError("Bad item"))
+ results.append(o)
+ return None
+
+ yield parallel_async(self.get_async_iterable(4), count, callable_)
+ assert sorted(results) == [0, 2, 3]
+ assert "Error processing an item" in caplog.text
+ assert "Bad item" in caplog.text
+
class TestDeferredFromCoro:
def test_deferred(self):