Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve sleep calculation in AsyncioLoopingCall. (#8294)

Andrey Rakhmatullindbcb9e04b4db
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index ba232459a..7116b449f 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import logging
+import math
import time
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from typing import (
@@ -181,24 +182,32 @@ class AsyncioLoopingCall(Generic[_P, _T]):
raise ValueError("Interval must be greater than 0")
self.interval = interval
- self._start_time = time.monotonic()
+ self._start_time = start_time = time.monotonic()
if now:
self._call()
loop = asyncio.get_event_loop()
- self._task = loop.create_task(self._loop())
+ self._task = loop.create_task(self._loop(start_time))
- def _to_sleep(self) -> float:
- """Return the time to sleep until the next call."""
+ def _next_boundary(self, boundary: float) -> float:
+ """Return the first interval boundary that is later than both
+ *boundary* and the current time.
+
+ The event loop may wake the looping call up slightly before
+ :func:`time.monotonic` reaches a boundary, as its timers use
+ ``loop.time()``, which may have a coarser resolution.
+
+ See also https://github.com/tornadoweb/tornado/issues/2333.
+ """
assert self.interval is not None
- assert self._start_time is not None
- now = time.monotonic()
- running_for = now - self._start_time
- return self.interval - (running_for % self.interval)
+ passed = max(math.floor((time.monotonic() - boundary) / self.interval), 0)
+ return boundary + (passed + 1) * self.interval
- async def _loop(self) -> None:
- """Run an infinite loop that calls the function periodically."""
+ async def _loop(self, boundary: float) -> None:
+ """Run an infinite loop that calls the function periodically, on the
+ interval boundaries that follow *boundary*."""
while self.running:
- await asyncio.sleep(self._to_sleep())
+ boundary = self._next_boundary(boundary)
+ await asyncio.sleep(boundary - time.monotonic())
self._call()
def stop(self) -> None:
diff --git a/tests/AsyncCrawlerProcess/closespider_timeout_no_item.py b/tests/AsyncCrawlerProcess/closespider_timeout_no_item.py
new file mode 100644
index 000000000..5760bcbe4
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/closespider_timeout_no_item.py
@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+
+from scrapy import Spider
+from scrapy.crawler import AsyncCrawlerProcess
+
+
+class ItemsSpider(Spider):
+ """Yields an item every 0.1 seconds for 3 seconds, so the spider must not
+ be closed by CLOSESPIDER_TIMEOUT_NO_ITEM = 1."""
+
+ name = "items"
+
+ async def start(self):
+ for i in range(30):
+ await asyncio.sleep(0.1)
+ yield {"i": i}
+
+
+if __name__ == "__main__":
+ ASYNCIO_EVENT_LOOP: str | None
+ try:
+ ASYNCIO_EVENT_LOOP = sys.argv[1]
+ except IndexError:
+ ASYNCIO_EVENT_LOOP = None
+
+ process = AsyncCrawlerProcess(
+ settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
+ "CLOSESPIDER_TIMEOUT_NO_ITEM": 1,
+ }
+ )
+ process.crawl(ItemsSpider)
+ process.start()
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 7049bd489..6c03168e6 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -525,6 +525,21 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
+ def test_custom_loop_closespider_timeout_no_item(self) -> None:
+ log = self.run_script("closespider_timeout_no_item.py")
+ assert "Closing spider since no items were produced" not in log
+ assert "Spider closed (finished)" in log
+ assert "'item_scraped_count': 30" in log
+
+ @pytest.mark.requires_uvloop
+ def test_uvloop_closespider_timeout_no_item(self) -> None:
+ # uvloop has a different clock resolution
+ log = self.run_script("closespider_timeout_no_item.py", "uvloop.Loop")
+ assert "Using asyncio event loop: uvloop.Loop" in log
+ assert "Closing spider since no items were produced" not in log
+ assert "Spider closed (finished)" in log
+ assert "'item_scraped_count': 30" in log
+
class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerRunner and AsyncCrawlerRunner,
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index d2deed61c..ed15bceae 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -2,12 +2,14 @@ from __future__ import annotations
import asyncio
import random
+from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
from twisted.internet.defer import Deferred
+import scrapy.utils.asyncio
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
@@ -19,7 +21,7 @@ from scrapy.utils.asyncio import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator
+ from collections.abc import AsyncGenerator, Callable, Iterator
@coroutine_test
@@ -129,6 +131,46 @@ class TestParallelAsyncio:
assert max(task_counts) < 100
+_real_sleep = asyncio.sleep
+
+
+class _FakeTime:
+ """A clock for :class:`~scrapy.utils.asyncio.AsyncioLoopingCall` that only
+ advances when the looping call sleeps or when a test advances it.
+
+ Each sleep ends *early* seconds before the requested time, like event loops
+ with timers coarser than :func:`time.monotonic` can do.
+ """
+
+ def __init__(self, *, early: float = 0.0):
+ self.now = 0.0
+ self._early = early
+
+ def monotonic(self) -> float:
+ return self.now
+
+ async def sleep(self, delay: float) -> None:
+ self.now += max(delay - self._early, 0.0)
+ await _real_sleep(0)
+
+ @contextmanager
+ def patch(self) -> Iterator[None]:
+ with (
+ mock.patch.object(scrapy.utils.asyncio, "time", self),
+ mock.patch.object(asyncio, "sleep", self.sleep),
+ ):
+ yield
+
+
+async def _wait_for(condition: Callable[[], bool]) -> None:
+ """Let other tasks run until *condition* is met, giving up after 1000
+ event loop iterations."""
+ for _ in range(1000):
+ if condition():
+ return
+ await _real_sleep(0)
+
+
@pytest.mark.only_asyncio
class TestAsyncioLoopingCall:
@coroutine_test
@@ -183,3 +225,37 @@ class TestAsyncioLoopingCall:
looping_call.start(0.1)
assert not looping_call.running
assert "Error calling the AsyncioLoopingCall function" in caplog.text
+
+ @coroutine_test
+ async def test_looping_call_early_wakeup(self) -> None:
+ """The function must be called exactly once per interval boundary even
+ if the event loop wakes the looping call before time.monotonic()
+ reaches the boundary (this can happen e.g. under uvloop)."""
+ fake_time = _FakeTime(early=0.001)
+ calls: list[float] = []
+ looping_call = AsyncioLoopingCall(lambda: calls.append(fake_time.now))
+ with fake_time.patch():
+ looping_call.start(1, now=False)
+ await _wait_for(lambda: len(calls) >= 5)
+ looping_call.stop()
+ assert calls == pytest.approx([0.999, 1.999, 2.999, 3.999, 4.999])
+
+ @coroutine_test
+ async def test_looping_call_late_wakeup(self) -> None:
+ """If the function is late (e.g. because the event loop was blocked),
+ the calls for the missed boundaries are skipped and the next call
+ happens on the next boundary, not immediately."""
+ fake_time = _FakeTime()
+ calls: list[float] = []
+
+ def func() -> None:
+ calls.append(fake_time.now)
+ if len(calls) == 1:
+ fake_time.now += 2.5 # the call blocks the loop for 2.5 intervals
+
+ looping_call = AsyncioLoopingCall(func)
+ with fake_time.patch():
+ looping_call.start(1, now=False)
+ await _wait_for(lambda: len(calls) >= 3)
+ looping_call.stop()
+ assert calls == [1, 4, 5]