Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add scrapy.utils.asyncio.sleep() (#7843)

Adrian1f03fbc17e1b
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 63c217e93..afccb491d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
Scrapy provides unified helpers for some of these examples:
+.. autofunction:: scrapy.utils.asyncio.sleep
.. autofunction:: scrapy.utils.asyncio.call_later
.. autofunction:: scrapy.utils.asyncio.create_looping_call
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index 44604c0fe..7c7697f56 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
from twisted.internet.defer import Deferred
-from twisted.internet.task import LoopingCall
+from twisted.internet.task import LoopingCall, deferLater
from twisted.internet.threads import deferToThread
from scrapy.utils.asyncgen import as_async_generator
@@ -293,6 +293,24 @@ class CallLaterResult:
self._delayed_call = None
+async def sleep(seconds: float) -> None:
+ """Sleep for *seconds*.
+
+ .. versionadded:: VERSION
+
+ This uses either :func:`asyncio.sleep` or
+ :func:`~twisted.internet.task.deferLater`, depending on whether asyncio
+ support is available.
+ """
+ if is_asyncio_available():
+ await asyncio.sleep(seconds)
+ return
+
+ from twisted.internet import reactor
+
+ await deferLater(reactor, seconds)
+
+
async def run_in_thread(
func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
) -> _T:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d0259b634..7c6235f29 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
from twisted.python import failure
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.asyncio import is_asyncio_available, sleep
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
@@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
- if is_asyncio_available():
- await asyncio.sleep(_DEFER_DELAY)
- else:
- from twisted.internet import reactor
-
- d: Deferred[None] = Deferred()
- reactor.callLater(_DEFER_DELAY, d.callback, None)
- await d
+ await sleep(_DEFER_DELAY)
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index fe2f83161..733b6797d 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from w3lib import __version__ as w3lib_version
-from tests.utils import async_sleep, get_script_run_env
+from scrapy.utils.asyncio import sleep
+from tests.utils import get_script_run_env
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
- await async_sleep(0.01)
+ await sleep(0.01)
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
p.wait() # type: ignore[no-untyped-call]
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index c15c396d3..14ec3d184 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
from scrapy.core.scheduler import BaseScheduler
-from scrapy.utils.asyncio import call_later
+from scrapy.utils.asyncio import call_later, sleep
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
-from tests.utils import async_sleep
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -65,23 +64,23 @@ class TestMain:
async def start(self):
yield Request("data:,a")
- await async_sleep(seconds)
+ await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
- await async_sleep(seconds)
+ await sleep(seconds)
self.crawler.engine._slot.scheduler.unpause()
# The scheduler request is processed.
- await async_sleep(seconds)
+ await sleep(seconds)
yield Request("data:,c")
- await async_sleep(seconds)
+ await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 9b7eb22fa..4bd54acd4 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
_parallel_asyncio,
+ call_later,
is_asyncio_available,
+ sleep,
)
from tests.utils.decorators import coroutine_test
@@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
assert is_asyncio_available() == (reactor_pytest != "default")
+@coroutine_test
+async def test_sleep() -> None:
+ events: list[str] = []
+ call_later(0.05, events.append, "call_later")
+ await sleep(0.1)
+ events.append("sleep")
+ assert events == ["call_later", "sleep"]
+
+
@pytest.mark.only_asyncio
class TestParallelAsyncio:
"""Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index b0632a7ea..b27c5ade7 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import os
from pathlib import Path
from typing import TYPE_CHECKING
@@ -8,8 +7,6 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy.settings import Settings, default_settings
-from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.defer import maybe_deferred_to_future
if TYPE_CHECKING:
from collections.abc import Callable
@@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):
return d
-async def async_sleep(seconds: float) -> None:
- if is_asyncio_available():
- await asyncio.sleep(seconds)
- else:
- await maybe_deferred_to_future(twisted_sleep(seconds))
-
-
def get_script_run_env() -> dict[str, str]:
"""Return a OS environment dict suitable to run scripts shipped with tests."""