Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle exceptions and cancellation in parallel helpers. (#8318)
Andrey Rakhmatullin2aa46683b396
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex 7116b449f..7fd4f7531 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -117,27 +117,25 @@ async def _parallel_asyncio(Tasks are created on demand, one per item, so that a *count* much largerthan the amount of work does not cost anything.- This function is only used in- :meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so it- assumes that neither *callable* nor iterating *iterable* will raise an- exception.+ Exceptions raised by *callable_* are logged. If this coroutine is+ cancelled, or iterating *iterable* raises an exception, the calls in+ progress are cancelled; an exception from *iterable* is raised in an+ :exc:`ExceptionGroup`."""semaphore = asyncio.Semaphore(count)- tasks: set[asyncio.Task[None]] = set()async def work(item: _T) -> None:try:await callable_(item, *args, **kwargs)+ except Exception:+ logger.exception("Error processing an item")finally:semaphore.release()- async for item in as_async_generator(iterable):- await semaphore.acquire()- task = asyncio.create_task(work(item))- tasks.add(task)- task.add_done_callback(tasks.discard)- if tasks:- await asyncio.wait(tasks)+ async with asyncio.TaskGroup() as tg:+ async for item in as_async_generator(iterable):+ await semaphore.acquire()+ tg.create_task(work(item))class AsyncioLoopingCall(Generic[_P, _T]):diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex d3fa4d214..4ce1a78fb 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -6,6 +6,7 @@ from __future__ import annotationsimport asyncioimport inspect+import loggingimport sysimport warningsfrom asyncio import Future@@ -48,6 +49,8 @@ if TYPE_CHECKING:request: Request+logger = logging.getLogger(__name__)+_T = TypeVar("_T")_T2 = TypeVar("_T2")_P = ParamSpec("_P")@@ -212,8 +215,20 @@ class _ParallelTasks:self._running += 1self._coop.coiterate(self._work).addBoth(self._task_done)- def _task_done(self, _: Any) -> None:+ def _task_done(self, result: Any) -> None:self._running -= 1+ if isinstance(result, failure.Failure):+ # circular import+ from scrapy.utils.log import failure_to_exc_info # noqa: PLC0415++ logger.error(+ "Error processing an item",+ exc_info=failure_to_exc_info(result),+ )+ # A failed task stops consuming work, which may not be exhausted+ # yet, so another one takes over.+ self.start()+ return# Only a running task can start another one, so once none is left no# more work can come.if not self._running:diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex ed15bceae..4fd53bd8c 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -130,6 +130,75 @@ class TestParallelAsyncio:assert results == [0, 1, 2]assert max(task_counts) < 100+ @pytest.mark.parametrize("count", [1, 2])+ @coroutine_test+ async def test_callable_error(+ self, count: int, caplog: pytest.LogCaptureFixture+ ) -> None:+ results: list[int] = []++ async def callable_(o: int) -> None:+ if o == 1:+ raise ValueError("Bad item")+ results.append(o)++ await _parallel_asyncio(range(4), count, callable_)+ assert sorted(results) == [0, 2, 3]+ assert "Error processing an item" in caplog.text+ assert "Bad item" in caplog.text++ @coroutine_test+ async def test_cancelled(self) -> None:+ """Cancelling the coroutine cancels the calls in progress."""+ started: list[int] = []+ cancelled: list[int] = []+ all_started = asyncio.Event()++ async def callable_(o: int) -> None:+ started.append(o)+ if len(started) == 3:+ all_started.set()+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled.append(o)+ raise++ task = asyncio.create_task(_parallel_asyncio(range(3), 3, callable_))+ await all_started.wait()+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert sorted(cancelled) == [0, 1, 2]++ @coroutine_test+ async def test_iteration_error(self) -> None:+ """An exception raised while iterating is propagated in an exception+ group, and the calls in progress are cancelled."""+ started: list[int] = []+ cancelled: list[int] = []+ all_started = asyncio.Event()++ async def iterable() -> AsyncGenerator[int, None]:+ yield 0+ yield 1+ await all_started.wait()+ raise ValueError("Bad iterable")++ async def callable_(o: int) -> None:+ started.append(o)+ if len(started) == 2:+ all_started.set()+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled.append(o)+ raise++ with pytest.RaisesGroup(pytest.RaisesExc(ValueError, match="Bad iterable")):+ await _parallel_asyncio(iterable(), 3, callable_)+ assert sorted(cancelled) == [0, 1]+_real_sleep = asyncio.sleepdiff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 5a277d08a..b10ddd25b 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Selffrom unittest import mockimport pytest-from twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.defer import Deferred, fail, inlineCallbacksfrom twisted.internet.interfaces import IReadDescriptorfrom twisted.internet.task import Cooperatorfrom zope.interface import implementer@@ -309,6 +309,24 @@ class TestParallelAsync:# One task per item, plus the one that finds no more work.assert coiterate.call_count <= 4+ @pytest.mark.parametrize("count", [1, 2])+ @inline_callbacks_test+ def test_callable_error(+ self, count: int, caplog: pytest.LogCaptureFixture+ ) -> Generator[Deferred[Any], Any, None]:+ results: list[int] = []++ def callable_(o: int) -> Deferred[None] | None:+ if o == 1:+ return fail(ValueError("Bad item"))+ results.append(o)+ return None++ yield parallel_async(self.get_async_iterable(4), count, callable_)+ assert sorted(results) == [0, 2, 3]+ assert "Error processing an item" in caplog.text+ assert "Bad item" in caplog.text+class TestDeferredFromCoro:def test_deferred(self):