Scrapy, a fast high-level web crawling & scraping framework for Python.
Add scrapy.utils.asyncio.sleep() (#7843)
Adrian1f03fbc17e1b
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex 63c217e93..afccb491d 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:Scrapy provides unified helpers for some of these examples:+.. autofunction:: scrapy.utils.asyncio.sleep.. autofunction:: scrapy.utils.asyncio.call_later.. autofunction:: scrapy.utils.asyncio.create_looping_call.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCalldiff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex 44604c0fe..7c7697f56 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterablefrom typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVarfrom twisted.internet.defer import Deferred-from twisted.internet.task import LoopingCall+from twisted.internet.task import LoopingCall, deferLaterfrom twisted.internet.threads import deferToThreadfrom scrapy.utils.asyncgen import as_async_generator@@ -293,6 +293,24 @@ class CallLaterResult:self._delayed_call = None+async def sleep(seconds: float) -> None:+ """Sleep for *seconds*.++ .. versionadded:: VERSION++ This uses either :func:`asyncio.sleep` or+ :func:`~twisted.internet.task.deferLater`, depending on whether asyncio+ support is available.+ """+ if is_asyncio_available():+ await asyncio.sleep(seconds)+ return++ from twisted.internet import reactor++ await deferLater(reactor, seconds)++async def run_in_thread(func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T:diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex d0259b634..7c6235f29 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -27,7 +27,7 @@ from twisted.internet.task import Cooperatorfrom twisted.python import failurefrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.asyncio import is_asyncio_available, sleepfrom scrapy.utils.python import global_object_nameif TYPE_CHECKING:@@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writersbefore attending pending delayed calls, so do not set delay to zero."""- if is_asyncio_available():- await asyncio.sleep(_DEFER_DELAY)- else:- from twisted.internet import reactor-- d: Deferred[None] = Deferred()- reactor.callLater(_DEFER_DELAY, d.callback, None)- await d+ await sleep(_DEFER_DELAY)def defer_result(result: Any) -> Deferred[Any]: # pragma: no coverdiff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex fe2f83161..733b6797d 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -14,7 +14,8 @@ from packaging.version import parse as parse_versionfrom pexpect.popen_spawn import PopenSpawnfrom w3lib import __version__ as w3lib_version-from tests.utils import async_sleep, get_script_run_env+from scrapy.utils.asyncio import sleep+from tests.utils import get_script_run_envfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):p.kill(sig)p.expect_exact("shutting down gracefully")# sending the second signal too fast often causes problems- await async_sleep(0.01)+ await sleep(0.01)p.kill(sig)p.expect_exact("forcing unclean shutdown")p.wait() # type: ignore[no-untyped-call]diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex c15c396d3..14ec3d184 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signalsfrom scrapy.core.scheduler import BaseScheduler-from scrapy.utils.asyncio import call_later+from scrapy.utils.asyncio import call_later, sleepfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer-from tests.utils import async_sleepfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -65,23 +64,23 @@ class TestMain:async def start(self):yield Request("data:,a")- await async_sleep(seconds)+ await sleep(seconds)self.crawler.engine._slot.scheduler.pause()self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))# During this time, the scheduler reports having requests but# returns None.- await async_sleep(seconds)+ await sleep(seconds)self.crawler.engine._slot.scheduler.unpause()# The scheduler request is processed.- await async_sleep(seconds)+ await sleep(seconds)yield Request("data:,c")- await async_sleep(seconds)+ await sleep(seconds)self.crawler.engine._slot.scheduler.pause()self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 9b7eb22fa..4bd54acd4 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.asyncio import (AsyncioLoopingCall,_parallel_asyncio,+ call_later,is_asyncio_available,+ sleep,)from tests.utils.decorators import coroutine_test@@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:assert is_asyncio_available() == (reactor_pytest != "default")+@coroutine_test+async def test_sleep() -> None:+ events: list[str] = []+ call_later(0.05, events.append, "call_later")+ await sleep(0.1)+ events.append("sleep")+ assert events == ["call_later", "sleep"]++@pytest.mark.only_asyncioclass TestParallelAsyncio:"""Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""diff --git a/tests/utils/__init__.py b/tests/utils/__init__.pyindex b0632a7ea..b27c5ade7 100644--- a/tests/utils/__init__.py+++ b/tests/utils/__init__.py@@ -1,6 +1,5 @@from __future__ import annotations-import asyncioimport osfrom pathlib import Pathfrom typing import TYPE_CHECKING@@ -8,8 +7,6 @@ from typing import TYPE_CHECKINGfrom twisted.internet.defer import Deferredfrom scrapy.settings import Settings, default_settings-from scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.defer import maybe_deferred_to_futureif TYPE_CHECKING:from collections.abc import Callable@@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):return d-async def async_sleep(seconds: float) -> None:- if is_asyncio_available():- await asyncio.sleep(seconds)- else:- await maybe_deferred_to_future(twisted_sleep(seconds))--def get_script_run_env() -> dict[str, str]:"""Return a OS environment dict suitable to run scripts shipped with tests."""