Scrapy, a fast high-level web crawling & scraping framework for Python.
Start item-processing workers on demand instead of one per CONCURRENT_ITEMS (#7959)
Adrianc8b09e5656db
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a8fe7d430..2837e498d 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -107,7 +107,9 @@ jobs:runs-on: ubuntu-latesttimeout-minutes: 60env:- PYTEST_ADDOPTS: ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}+ # A hanging test would otherwise keep the job running until GitHub kills+ # it, without telling which test hung.+ PYTEST_ADDOPTS: --timeout=120 ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}strategy:fail-fast: falsematrix:@@ -173,7 +175,9 @@ jobs:runs-on: ubuntu-latesttimeout-minutes: 60env:- PYTEST_ADDOPTS: ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}+ # A hanging test would otherwise keep the job running until GitHub kills+ # it, without telling which test hung.+ PYTEST_ADDOPTS: --timeout=120 ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}strategy:fail-fast: falsematrix:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 08eff6ead..64459e485 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -121,7 +121,7 @@ class Scraper:]:self._check_deprecated_itemproc_method(method)- self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")+ self.concurrent_items: int = max(1, crawler.settings.getint("CONCURRENT_ITEMS"))self.crawler: Crawler = crawlerself.signals: SignalManager = crawler.signalsself.logformatter: LogFormatter = crawler.logformatterdiff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex fff8c3f94..b65523f66 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -98,17 +98,6 @@ def is_asyncio_available() -> bool:return _is_asyncio_reactor_installed()-class _QueueEnd:- """Marks the end of the work queue of :func:`_parallel_asyncio`.-- A dedicated type is needed because any value, ``None`` included, can be an- item of the iterable being worked on.- """---_QUEUE_END = _QueueEnd()--async def _parallel_asyncio(iterable: Iterable[_T] | AsyncIterator[_T],count: int,@@ -119,32 +108,30 @@ async def _parallel_asyncio("""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.+ Tasks are created on demand, one per item, so that a *count* much larger+ than the amount of work does not cost anything.+This function is only used in:meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so itassumes that neither *callable* nor iterating *iterable* will raise anexception."""- queue: asyncio.Queue[_T | _QueueEnd] = asyncio.Queue(count * 2)-- async def worker() -> None:- while True:- item = await queue.get()- if isinstance(item, _QueueEnd):- break- try:- await callable_(item, *args, **kwargs)- finally:- queue.task_done()-- async def fill_queue() -> None:- async for item in as_async_generator(iterable):- await queue.put(item)- for _ in range(count):- await queue.put(_QUEUE_END)-- fill_task = asyncio.create_task(fill_queue())- work_tasks = [asyncio.create_task(worker()) for _ in range(count)]- await asyncio.wait([fill_task, *work_tasks])+ semaphore = asyncio.Semaphore(count)+ tasks: set[asyncio.Task[None]] = set()++ async def work(item: _T) -> None:+ try:+ await callable_(item, *args, **kwargs)+ finally:+ semaphore.release()++ async for item in as_async_generator(iterable):+ await semaphore.acquire()+ task = asyncio.create_task(work(item))+ tasks.add(task)+ task.add_done_callback(tasks.discard)+ if tasks:+ await asyncio.wait(tasks)class AsyncioLoopingCall(Generic[_P, _T]):diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex e7f5953e7..a002442cf 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -178,21 +178,56 @@ def mustbe_deferred(return defer_result(result)+class _ParallelTasks:+ """Consumes *work* with no more than *count* concurrent cooperative tasks.++ Tasks are started on demand: :meth:`start` must be called every time a new+ item becomes available, so that a *count* much larger than the amount of+ work does not cost anything.++ :attr:`finished` fires once *work* is exhausted and every task is done.+ """++ def __init__(self, work: Iterator[Any], count: int):+ self._coop = Cooperator()+ self._work = work+ self._count = count+ self._running = 0+ self.finished: Deferred[None] = Deferred()++ def start(self) -> None:+ if self._running >= self._count:+ return+ self._running += 1+ self._coop.coiterate(self._work).addBoth(self._task_done)++ def _task_done(self, _: Any) -> None:+ self._running -= 1+ # Only a running task can start another one, so once none is left no+ # more work can come.+ if not self._running:+ self.finished.callback(None)++def parallel(iterable: Iterable[_T],count: int,callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002*args: _P.args,**named: _P.kwargs,-) -> Deferred[list[tuple[bool, Iterator[_T2]]]]:+) -> Deferred[None]:"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.-- Taken from: https://jcalderone.livejournal.com/24285.html"""- coop = Cooperator()- work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable)- return DeferredList([coop.coiterate(work) for _ in range(count)])++ def work() -> Iterator[_T2]:+ for elem in iterable:+ tasks.start()+ yield callable(elem, *args, **named)++ tasks = _ParallelTasks(work(), count)+ tasks.start()+ return tasks.finishedclass _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):@@ -255,12 +290,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):self.finished: bool = Falseself.waiting_deferreds: deque[Deferred[Any]] = deque()self.anext_deferred: Deferred[_T] | None = None+ # Called whenever aiterator produces a value, so that parallel_async()+ # can start a task for it.+ self.on_value: Callable[[], None] = lambda: Nonedef _callback(self, result: _T) -> None:# This gets called when the result from aiterator.__anext__() is available.# It calls the callable on it and sends the result to the oldest waiting Deferred# (by chaining if the result is a Deferred too or by firing if not).self.anext_deferred = None+ self.on_value()callable_result = self.callable(result, *self.callable_args, **self.callable_kwargs)@@ -305,16 +344,13 @@ def parallel_async(callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], # noqa: A002*args: _P.args,**named: _P.kwargs,-) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]:+) -> Deferred[None]:"""Like ``parallel`` but for async iterators"""- coop = Cooperator()- work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter(- async_iterable, callable, *args, **named- )- dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList(- [coop.coiterate(work) for _ in range(count)]- )- return dl+ work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)+ tasks = _ParallelTasks(work, count)+ work.on_value = tasks.start+ tasks.start()+ return tasks.finisheddef process_chain(diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex b66852b99..f9be16ba0 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -115,6 +115,19 @@ class TestParallelAsyncio:assert parallel_count[0] == 0assert max_parallel_count[0] <= self.CONCURRENT_ITEMS+ @coroutine_test+ async def test_count_higher_than_work(self):+ results: list[int] = []+ task_counts: list[int] = []++ async def callable_(o: int) -> None:+ task_counts.append(len(asyncio.all_tasks()))+ results.append(o)++ await _parallel_asyncio(range(3), 1_000_000, callable_)+ assert results == [0, 1, 2]+ assert max(task_counts) < 100+@pytest.mark.only_asyncioclass TestAsyncioLoopingCall:diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 17a80aa80..0e8daeb1b 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -6,10 +6,12 @@ import socketimport warningsfrom asyncio import Futurefrom typing import TYPE_CHECKING, Any+from unittest import mockimport pytestfrom twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.internet.interfaces import IReadDescriptor+from twisted.internet.task import Cooperatorfrom zope.interface import implementerfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgen@@ -24,6 +26,7 @@ from scrapy.utils.defer import (maybe_deferred_to_future,maybeDeferred_coro,mustbe_deferred,+ parallel,parallel_async,)from tests.utils.decorators import coroutine_test, inline_callbacks_test@@ -137,6 +140,20 @@ class TestAsyncDefTestsuite:raise RuntimeError("This is expected to be raised")[email protected]_reactor # parallel() requires a reactor+class TestParallel:+ @inline_callbacks_test+ def test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:+ results: list[int] = []+ with mock.patch.object(+ Cooperator, "coiterate", autospec=True, side_effect=Cooperator.coiterate+ ) as coiterate:+ yield parallel(range(3), 1_000_000, results.append)+ assert results == [0, 1, 2]+ # One task per item, plus the one that finds no more work.+ assert coiterate.call_count <= 4++@implementer(IReadDescriptor)class _ReadTracker:"""Socket that the event loop finds readable, counting how often it reads it.@@ -322,6 +339,17 @@ class TestParallelAsync:assert parallel_count[0] == 0assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]+ @inline_callbacks_test+ def test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:+ results: list[int] = []+ with mock.patch.object(+ Cooperator, "coiterate", autospec=True, side_effect=Cooperator.coiterate+ ) as coiterate:+ yield parallel_async(self.get_async_iterable(3), 1_000_000, results.append)+ assert sorted(results) == [0, 1, 2]+ # One task per item, plus the one that finds no more work.+ assert coiterate.call_count <= 4+class TestDeferredFromCoro:def test_deferred(self):