Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve sleep calculation in AsyncioLoopingCall. (#8294)
Andrey Rakhmatullindbcb9e04b4db
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex ba232459a..7116b449f 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport asyncioimport logging+import mathimport timefrom collections.abc import AsyncIterator, Callable, Coroutine, Iterablefrom typing import (@@ -181,24 +182,32 @@ class AsyncioLoopingCall(Generic[_P, _T]):raise ValueError("Interval must be greater than 0")self.interval = interval- self._start_time = time.monotonic()+ self._start_time = start_time = time.monotonic()if now:self._call()loop = asyncio.get_event_loop()- self._task = loop.create_task(self._loop())+ self._task = loop.create_task(self._loop(start_time))- def _to_sleep(self) -> float:- """Return the time to sleep until the next call."""+ def _next_boundary(self, boundary: float) -> float:+ """Return the first interval boundary that is later than both+ *boundary* and the current time.++ The event loop may wake the looping call up slightly before+ :func:`time.monotonic` reaches a boundary, as its timers use+ ``loop.time()``, which may have a coarser resolution.++ See also https://github.com/tornadoweb/tornado/issues/2333.+ """assert self.interval is not None- assert self._start_time is not None- now = time.monotonic()- running_for = now - self._start_time- return self.interval - (running_for % self.interval)+ passed = max(math.floor((time.monotonic() - boundary) / self.interval), 0)+ return boundary + (passed + 1) * self.interval- async def _loop(self) -> None:- """Run an infinite loop that calls the function periodically."""+ async def _loop(self, boundary: float) -> None:+ """Run an infinite loop that calls the function periodically, on the+ interval boundaries that follow *boundary*."""while self.running:- await asyncio.sleep(self._to_sleep())+ boundary = self._next_boundary(boundary)+ await asyncio.sleep(boundary - time.monotonic())self._call()def stop(self) -> None:diff --git a/tests/AsyncCrawlerProcess/closespider_timeout_no_item.py b/tests/AsyncCrawlerProcess/closespider_timeout_no_item.pynew file mode 100644index 000000000..5760bcbe4--- /dev/null+++ b/tests/AsyncCrawlerProcess/closespider_timeout_no_item.py@@ -0,0 +1,37 @@+from __future__ import annotations++import asyncio+import sys++from scrapy import Spider+from scrapy.crawler import AsyncCrawlerProcess+++class ItemsSpider(Spider):+ """Yields an item every 0.1 seconds for 3 seconds, so the spider must not+ be closed by CLOSESPIDER_TIMEOUT_NO_ITEM = 1."""++ name = "items"++ async def start(self):+ for i in range(30):+ await asyncio.sleep(0.1)+ yield {"i": i}+++if __name__ == "__main__":+ ASYNCIO_EVENT_LOOP: str | None+ try:+ ASYNCIO_EVENT_LOOP = sys.argv[1]+ except IndexError:+ ASYNCIO_EVENT_LOOP = None++ process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,+ "CLOSESPIDER_TIMEOUT_NO_ITEM": 1,+ }+ )+ process.crawl(ItemsSpider)+ process.start()diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 7049bd489..6c03168e6 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -525,6 +525,21 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):in log)+ def test_custom_loop_closespider_timeout_no_item(self) -> None:+ log = self.run_script("closespider_timeout_no_item.py")+ assert "Closing spider since no items were produced" not in log+ assert "Spider closed (finished)" in log+ assert "'item_scraped_count': 30" in log++ @pytest.mark.requires_uvloop+ def test_uvloop_closespider_timeout_no_item(self) -> None:+ # uvloop has a different clock resolution+ log = self.run_script("closespider_timeout_no_item.py", "uvloop.Loop")+ assert "Using asyncio event loop: uvloop.Loop" in log+ assert "Closing spider since no items were produced" not in log+ assert "Spider closed (finished)" in log+ assert "'item_scraped_count': 30" in log+class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):"""Common tests between CrawlerRunner and AsyncCrawlerRunner,diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex d2deed61c..ed15bceae 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -2,12 +2,14 @@ from __future__ import annotationsimport asyncioimport random+from contextlib import contextmanagerfrom typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytestfrom twisted.internet.defer import Deferred+import scrapy.utils.asynciofrom scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.asyncio import (AsyncioLoopingCall,@@ -19,7 +21,7 @@ from scrapy.utils.asyncio import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from collections.abc import AsyncGenerator+ from collections.abc import AsyncGenerator, Callable, Iterator@coroutine_test@@ -129,6 +131,46 @@ class TestParallelAsyncio:assert max(task_counts) < 100+_real_sleep = asyncio.sleep+++class _FakeTime:+ """A clock for :class:`~scrapy.utils.asyncio.AsyncioLoopingCall` that only+ advances when the looping call sleeps or when a test advances it.++ Each sleep ends *early* seconds before the requested time, like event loops+ with timers coarser than :func:`time.monotonic` can do.+ """++ def __init__(self, *, early: float = 0.0):+ self.now = 0.0+ self._early = early++ def monotonic(self) -> float:+ return self.now++ async def sleep(self, delay: float) -> None:+ self.now += max(delay - self._early, 0.0)+ await _real_sleep(0)++ @contextmanager+ def patch(self) -> Iterator[None]:+ with (+ mock.patch.object(scrapy.utils.asyncio, "time", self),+ mock.patch.object(asyncio, "sleep", self.sleep),+ ):+ yield+++async def _wait_for(condition: Callable[[], bool]) -> None:+ """Let other tasks run until *condition* is met, giving up after 1000+ event loop iterations."""+ for _ in range(1000):+ if condition():+ return+ await _real_sleep(0)++@pytest.mark.only_asyncioclass TestAsyncioLoopingCall:@coroutine_test@@ -183,3 +225,37 @@ class TestAsyncioLoopingCall:looping_call.start(0.1)assert not looping_call.runningassert "Error calling the AsyncioLoopingCall function" in caplog.text++ @coroutine_test+ async def test_looping_call_early_wakeup(self) -> None:+ """The function must be called exactly once per interval boundary even+ if the event loop wakes the looping call before time.monotonic()+ reaches the boundary (this can happen e.g. under uvloop)."""+ fake_time = _FakeTime(early=0.001)+ calls: list[float] = []+ looping_call = AsyncioLoopingCall(lambda: calls.append(fake_time.now))+ with fake_time.patch():+ looping_call.start(1, now=False)+ await _wait_for(lambda: len(calls) >= 5)+ looping_call.stop()+ assert calls == pytest.approx([0.999, 1.999, 2.999, 3.999, 4.999])++ @coroutine_test+ async def test_looping_call_late_wakeup(self) -> None:+ """If the function is late (e.g. because the event loop was blocked),+ the calls for the missed boundaries are skipped and the next call+ happens on the next boundary, not immediately."""+ fake_time = _FakeTime()+ calls: list[float] = []++ def func() -> None:+ calls.append(fake_time.now)+ if len(calls) == 1:+ fake_time.now += 2.5 # the call blocks the loop for 2.5 intervals++ looping_call = AsyncioLoopingCall(func)+ with fake_time.patch():+ looping_call.start(1, now=False)+ await _wait_for(lambda: len(calls) >= 3)+ looping_call.stop()+ assert calls == [1, 4, 5]