Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop sleeping 100 ms before every callback and media result (#8019)
Adriane01783a493b7
diff --git a/conftest.py b/conftest.pyindex 1a1fcf053..bd952476f 100644--- a/conftest.py+++ b/conftest.py@@ -111,11 +111,12 @@ def reactor_pytest(request) -> str:def pytest_configure(config):- if config.getoption("--reactor") == "asyncio":- # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.- # If we decide to run tests with both, we will need to add a new option and check it here.+ if config.getoption("--reactor") in {"asyncio", "none"}:+ # Needed on Windows to switch from proactor to selector, which supports+ # add_reader/add_writer (required by the Twisted asyncio reactor, and by+ # tests that register their own readers) and is what Twisted expects.set_asyncio_event_loop_policy()- elif config.getoption("--reactor") == "none":+ if config.getoption("--reactor") == "none":install_reactor_import_hook()diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 2089b1224..c695a0786 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -22,7 +22,7 @@ from scrapy.utils.asyncio import ()from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (- _defer_sleep_async,+ _process_pending_io,_schedule_coro,deferred_from_coro,maybe_deferred_to_future,@@ -219,7 +219,7 @@ class Downloader:)return responseexcept Exception:- await _defer_sleep_async()+ await _process_pending_io()raisefinally:# 3. After response arrives, remove the request from transferringdiff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex ab74e22a4..a71589d4a 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -15,7 +15,7 @@ from scrapy.http import Request, Responsefrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import (- _defer_sleep_async,+ _process_pending_io,deferred_from_coro,ensure_awaitable,maybe_deferred_to_future,@@ -81,7 +81,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):request, download_func)except Exception as ex:- await _defer_sleep_async()+ await _process_pending_io()# either returns a request or response (which we pass to process_response())# or reraises the exceptionresult = await self._process_exception(ex, request)diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 8146f0e15..2d31d31ac 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -24,7 +24,7 @@ from scrapy.http import Request, Responsefrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_availablefrom scrapy.utils.defer import (- _defer_sleep_async,+ _process_pending_io,_schedule_coro,aiter_errback,deferred_from_coro,@@ -312,7 +312,7 @@ class Scraper:.. versionadded:: 2.13"""- await _defer_sleep_async()+ await _process_pending_io()assert self.crawler.spiderif isinstance(result, Response):if getattr(result, "request", None) is None:diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 5d1ec246c..8d9aba60f 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -23,7 +23,7 @@ from scrapy.middleware import MiddlewareManagerfrom scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import (- _defer_sleep_async,+ _process_pending_io,deferred_from_coro,maybe_deferred_to_future,)@@ -224,7 +224,7 @@ class SpiderMiddlewareManager(MiddlewareManager):ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)return await self._process_callback_output(response, ait)except Exception as ex:- await _defer_sleep_async()+ await _process_pending_io()return self._process_spider_exception(response, ex)async def process_start(diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex d4025bf3b..41b2e17c9 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -14,12 +14,11 @@ from twisted.python.failure import Failurefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.request import NO_CALLBACK, Requestfrom scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACK-from scrapy.utils.asyncio import call_later, is_asyncio_available+from scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (- _DEFER_DELAY,- _defer_sleep_async,+ _process_pending_io,deferred_from_coro,ensure_awaitable,maybe_deferred_to_future,@@ -177,7 +176,7 @@ class MediaPipeline(ABC):# Return cached result if request was already seenif fp in info.downloaded:- await _defer_sleep_async()+ await _process_pending_io()cached_result = info.downloaded[fp]if isinstance(cached_result, Failure):if eb:@@ -197,7 +196,7 @@ class MediaPipeline(ABC):# Download request checking media_to_download hook output firstinfo.downloading.add(fp)- await _defer_sleep_async()+ await _process_pending_io()result: FileInfo | Failuretry:file_info: FileInfo | None = await ensure_awaitable(@@ -213,6 +212,7 @@ class MediaPipeline(ABC):result = Failure()if not _media_request_filtered(result):logger.exception(result)+ await _process_pending_io()self._cache_result_and_execute_waiters(result, fp, info)return await maybe_deferred_to_future(wad) # it must return wad at last@@ -278,9 +278,9 @@ class MediaPipeline(ABC):info.downloaded[fp] = result # cache resultfor wad in info.waiting.pop(fp):if isinstance(result, Failure):- call_later(_DEFER_DELAY, wad.errback, result)+ wad.errback(result)else:- call_later(_DEFER_DELAY, wad.callback, result)+ wad.callback(result)# Overridable Interface@abstractmethoddiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 7c6235f29..2b5e3f702 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -86,11 +86,15 @@ def defer_succeed(result: _T) -> Deferred[_T]: # pragma: no coverreturn d-async def _defer_sleep_async() -> None:- """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers- before attending pending delayed calls, so do not set delay to zero.+async def _process_pending_io() -> None:+ """Yield control until the event loop has gone through its readers and writers.++ Yielding twice is what makes that guarantee: the first yield can resume+ before the callbacks of the file descriptors that the poll found ready, and+ only the second one is certain to resume after them."""- await sleep(_DEFER_DELAY)+ await sleep(0)+ await sleep(0)def defer_result(result: Any) -> Deferred[Any]: # pragma: no coverdiff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 30a42da48..fd377fb7f 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -17,7 +17,7 @@ from scrapy.pipelines.media import (MediaPipeline,_MediaRequestFiltered,)-from scrapy.utils.defer import _defer_sleep_async+from scrapy.utils.defer import _process_pending_iofrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.signal import disconnect_all@@ -443,7 +443,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):rsp1 = Response("http://url")async def rsp1_func():- await _defer_sleep_async()+ await _process_pending_io()_check_downloading(rsp1)async def rsp2_func():diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex e61741fec..14eea068c 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -2,15 +2,20 @@ from __future__ import annotationsimport asyncioimport random+import socketimport warningsfrom asyncio import Futurefrom typing import TYPE_CHECKING, Anyimport pytestfrom twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.interfaces import IReadDescriptor+from zope.interface import implementerfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgen+from scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import (+ _process_pending_io,aiter_errback,deferred_f_from_coro_f,deferred_from_coro,@@ -27,6 +32,7 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Generatorfrom twisted.python.failure import Failure+ from typing_extensions import Self@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor@@ -131,6 +137,81 @@ class TestAsyncDefTestsuite:raise RuntimeError("This is expected to be raised")+@implementer(IReadDescriptor)+class _ReadTracker:+ """Socket that the event loop finds readable, counting how often it reads it.++ Entering the context manager registers the socket and makes it readable,+ from the next poll of the event loop on, so that a non-zero *reads* means+ that the loop has polled its file descriptors since then. *on_read*, if+ given, is called on every read.+ """++ def __init__(self, on_read: Callable[[], None] | None = None) -> None:+ self._on_read = on_read+ self._rx, self._tx = socket.socketpair()+ self.reads = 0++ def _read(self) -> None:+ self.reads += 1+ self._rx.recv(1)+ if self._on_read is not None:+ self._on_read()++ def fileno(self) -> int:+ return self._rx.fileno()++ doRead = _read++ def connectionLost(self, reason: Failure) -> None:+ pass++ def logPrefix(self) -> str:+ return "read-tracker"++ def __enter__(self) -> Self:+ if is_asyncio_available():+ asyncio.get_event_loop().add_reader(self._rx, self._read)+ else:+ from twisted.internet import reactor++ reactor.addReader(self)+ self._tx.send(b"x")+ return self++ def __exit__(self, *exc_info: object) -> None:+ if is_asyncio_available():+ asyncio.get_event_loop().remove_reader(self._rx)+ else:+ from twisted.internet import reactor++ reactor.removeReader(self)+ self._rx.close()+ self._tx.close()+++@coroutine_test+async def test_process_pending_io() -> None:+ # The wait runs from a read callback, which is where Scrapy's own waits+ # happen, since what precedes them is a response arriving over a socket. It+ # is also the strictest place to wait from: the event loop polled right+ # before calling us, so only another poll can read the socket that check()+ # registers.+ done: Deferred[int] = Deferred()++ async def check() -> None:+ with _ReadTracker() as tracker:+ await _process_pending_io()+ done.callback(tracker.reads)++ def start() -> None:+ deferred_from_coro(check()).addErrback(done.errback)++ with _ReadTracker(on_read=start):+ reads = await maybe_deferred_to_future(done)+ assert reads++@pytest.mark.requires_reactor # parallel_async() requires a reactorclass TestParallelAsync:"""This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.