Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop sleeping 100 ms before every callback and media result (#8019)

Adriane01783a493b7
diff --git a/conftest.py b/conftest.py
index 1a1fcf053..bd952476f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -111,11 +111,12 @@ def reactor_pytest(request) -> str:
def pytest_configure(config):
- if config.getoption("--reactor") == "asyncio":
- # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
- # If we decide to run tests with both, we will need to add a new option and check it here.
+ if config.getoption("--reactor") in {"asyncio", "none"}:
+ # Needed on Windows to switch from proactor to selector, which supports
+ # add_reader/add_writer (required by the Twisted asyncio reactor, and by
+ # tests that register their own readers) and is what Twisted expects.
set_asyncio_event_loop_policy()
- elif config.getoption("--reactor") == "none":
+ if config.getoption("--reactor") == "none":
install_reactor_import_hook()
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 2089b1224..c695a0786 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -22,7 +22,7 @@ from scrapy.utils.asyncio import (
)
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
- _defer_sleep_async,
+ _process_pending_io,
_schedule_coro,
deferred_from_coro,
maybe_deferred_to_future,
@@ -219,7 +219,7 @@ class Downloader:
)
return response
except Exception:
- await _defer_sleep_async()
+ await _process_pending_io()
raise
finally:
# 3. After response arrives, remove the request from transferring
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index ab74e22a4..a71589d4a 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -15,7 +15,7 @@ from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
- _defer_sleep_async,
+ _process_pending_io,
deferred_from_coro,
ensure_awaitable,
maybe_deferred_to_future,
@@ -81,7 +81,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
request, download_func
)
except Exception as ex:
- await _defer_sleep_async()
+ await _process_pending_io()
# either returns a request or response (which we pass to process_response())
# or reraises the exception
result = await self._process_exception(ex, request)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8146f0e15..2d31d31ac 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -24,7 +24,7 @@ from scrapy.http import Request, Response
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
from scrapy.utils.defer import (
- _defer_sleep_async,
+ _process_pending_io,
_schedule_coro,
aiter_errback,
deferred_from_coro,
@@ -312,7 +312,7 @@ class Scraper:
.. versionadded:: 2.13
"""
- await _defer_sleep_async()
+ await _process_pending_io()
assert self.crawler.spider
if isinstance(result, Response):
if getattr(result, "request", None) is None:
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 5d1ec246c..8d9aba60f 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -23,7 +23,7 @@ from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
- _defer_sleep_async,
+ _process_pending_io,
deferred_from_coro,
maybe_deferred_to_future,
)
@@ -224,7 +224,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)
return await self._process_callback_output(response, ait)
except Exception as ex:
- await _defer_sleep_async()
+ await _process_pending_io()
return self._process_spider_exception(response, ex)
async def process_start(
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index d4025bf3b..41b2e17c9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -14,12 +14,11 @@ from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import NO_CALLBACK, Request
from scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACK
-from scrapy.utils.asyncio import call_later, is_asyncio_available
+from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
- _DEFER_DELAY,
- _defer_sleep_async,
+ _process_pending_io,
deferred_from_coro,
ensure_awaitable,
maybe_deferred_to_future,
@@ -177,7 +176,7 @@ class MediaPipeline(ABC):
# Return cached result if request was already seen
if fp in info.downloaded:
- await _defer_sleep_async()
+ await _process_pending_io()
cached_result = info.downloaded[fp]
if isinstance(cached_result, Failure):
if eb:
@@ -197,7 +196,7 @@ class MediaPipeline(ABC):
# Download request checking media_to_download hook output first
info.downloading.add(fp)
- await _defer_sleep_async()
+ await _process_pending_io()
result: FileInfo | Failure
try:
file_info: FileInfo | None = await ensure_awaitable(
@@ -213,6 +212,7 @@ class MediaPipeline(ABC):
result = Failure()
if not _media_request_filtered(result):
logger.exception(result)
+ await _process_pending_io()
self._cache_result_and_execute_waiters(result, fp, info)
return await maybe_deferred_to_future(wad) # it must return wad at last
@@ -278,9 +278,9 @@ class MediaPipeline(ABC):
info.downloaded[fp] = result # cache result
for wad in info.waiting.pop(fp):
if isinstance(result, Failure):
- call_later(_DEFER_DELAY, wad.errback, result)
+ wad.errback(result)
else:
- call_later(_DEFER_DELAY, wad.callback, result)
+ wad.callback(result)
# Overridable Interface
@abstractmethod
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 7c6235f29..2b5e3f702 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -86,11 +86,15 @@ def defer_succeed(result: _T) -> Deferred[_T]: # pragma: no cover
return d
-async def _defer_sleep_async() -> None:
- """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
- before attending pending delayed calls, so do not set delay to zero.
+async def _process_pending_io() -> None:
+ """Yield control until the event loop has gone through its readers and writers.
+
+ Yielding twice is what makes that guarantee: the first yield can resume
+ before the callbacks of the file descriptors that the poll found ready, and
+ only the second one is certain to resume after them.
"""
- await sleep(_DEFER_DELAY)
+ await sleep(0)
+ await sleep(0)
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 30a42da48..fd377fb7f 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -17,7 +17,7 @@ from scrapy.pipelines.media import (
MediaPipeline,
_MediaRequestFiltered,
)
-from scrapy.utils.defer import _defer_sleep_async
+from scrapy.utils.defer import _process_pending_io
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.signal import disconnect_all
@@ -443,7 +443,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
rsp1 = Response("http://url")
async def rsp1_func():
- await _defer_sleep_async()
+ await _process_pending_io()
_check_downloading(rsp1)
async def rsp2_func():
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index e61741fec..14eea068c 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -2,15 +2,20 @@ from __future__ import annotations
import asyncio
import random
+import socket
import warnings
from asyncio import Future
from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.interfaces import IReadDescriptor
+from zope.interface import implementer
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import (
+ _process_pending_io,
aiter_errback,
deferred_f_from_coro_f,
deferred_from_coro,
@@ -27,6 +32,7 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
from twisted.python.failure import Failure
+ from typing_extensions import Self
@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor
@@ -131,6 +137,81 @@ class TestAsyncDefTestsuite:
raise RuntimeError("This is expected to be raised")
+@implementer(IReadDescriptor)
+class _ReadTracker:
+ """Socket that the event loop finds readable, counting how often it reads it.
+
+ Entering the context manager registers the socket and makes it readable,
+ from the next poll of the event loop on, so that a non-zero *reads* means
+ that the loop has polled its file descriptors since then. *on_read*, if
+ given, is called on every read.
+ """
+
+ def __init__(self, on_read: Callable[[], None] | None = None) -> None:
+ self._on_read = on_read
+ self._rx, self._tx = socket.socketpair()
+ self.reads = 0
+
+ def _read(self) -> None:
+ self.reads += 1
+ self._rx.recv(1)
+ if self._on_read is not None:
+ self._on_read()
+
+ def fileno(self) -> int:
+ return self._rx.fileno()
+
+ doRead = _read
+
+ def connectionLost(self, reason: Failure) -> None:
+ pass
+
+ def logPrefix(self) -> str:
+ return "read-tracker"
+
+ def __enter__(self) -> Self:
+ if is_asyncio_available():
+ asyncio.get_event_loop().add_reader(self._rx, self._read)
+ else:
+ from twisted.internet import reactor
+
+ reactor.addReader(self)
+ self._tx.send(b"x")
+ return self
+
+ def __exit__(self, *exc_info: object) -> None:
+ if is_asyncio_available():
+ asyncio.get_event_loop().remove_reader(self._rx)
+ else:
+ from twisted.internet import reactor
+
+ reactor.removeReader(self)
+ self._rx.close()
+ self._tx.close()
+
+
+@coroutine_test
+async def test_process_pending_io() -> None:
+ # The wait runs from a read callback, which is where Scrapy's own waits
+ # happen, since what precedes them is a response arriving over a socket. It
+ # is also the strictest place to wait from: the event loop polled right
+ # before calling us, so only another poll can read the socket that check()
+ # registers.
+ done: Deferred[int] = Deferred()
+
+ async def check() -> None:
+ with _ReadTracker() as tracker:
+ await _process_pending_io()
+ done.callback(tracker.reads)
+
+ def start() -> None:
+ deferred_from_coro(check()).addErrback(done.errback)
+
+ with _ReadTracker(on_read=start):
+ reads = await maybe_deferred_to_future(done)
+ assert reads
+
+
@pytest.mark.requires_reactor # parallel_async() requires a reactor
class TestParallelAsync:
"""This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.