Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Force download timeouts in BaseStreamingDownloadHandler. (#8481)

Andrey Rakhmatullinaeaec6a5bd84
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8465d4697..bf663a05f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1248,9 +1248,12 @@ Default: ``180``
The amount of time (in secs) that the downloader will wait before timing out.
+A value of ``0`` or ``None`` doesn't mean a timeout of 0 or infinity: a
+non-zero handler-specific value will be used instead.
+
.. note::
- This timeout can be per-request using the :reqmeta:`download_timeout`
+ This timeout can be set per-request using the :reqmeta:`download_timeout`
:attr:`.Request.meta` key.
.. note::
diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.py
index ea4b661bb..3903efcb6 100644
--- a/scrapy/core/downloader/handlers/_aioftp.py
+++ b/scrapy/core/downloader/handlers/_aioftp.py
@@ -218,9 +218,9 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):
return super()._get_body(buffer, response, request)
async def _read_response(
- self, response: _AioftpResponse, request: Request
+ self, response: _AioftpResponse, request: Request, deadline: float
) -> Response:
- result = await super()._read_response(response, request)
+ result = await super()._read_response(response, request, deadline)
if response.data is not None:
self._add_ftp_headers(result, response, request)
return result
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 615ec4391..799126ad0 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -1,9 +1,11 @@
from __future__ import annotations
+import asyncio
import base64
import logging
import time
from abc import abstractmethod
+from contextlib import AsyncExitStack
from io import BytesIO
from pathlib import Path
from typing import (
@@ -24,6 +26,7 @@ from w3lib.url import add_http_if_no_scheme
from scrapy import Request, signals
from scrapy.exceptions import (
DownloadCancelledError,
+ DownloadTimeoutError,
NotConfigured,
ResponseDataLossError,
)
@@ -72,13 +75,12 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
_DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10
experimental: ClassVar[bool] = False
- requires_asyncio: ClassVar[bool] = True
# require subclasses to disable proxies explicitly with an explanation
supports_proxies: ClassVar[bool] = True
supports_per_request_bindaddress: ClassVar[bool] = False
def __init__(self, crawler: Crawler):
- if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover
+ if not is_asyncio_available(): # pragma: no cover
raise NotConfigured(f"{type(self).__name__} requires the asyncio support.")
self._check_deps_installed()
super().__init__(crawler)
@@ -167,15 +169,27 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
f"The 'bindaddress' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
- timeout: float = request.meta.get(
- "download_timeout", self._DEFAULT_CONNECT_TIMEOUT
+ timeout: float = (
+ request.meta.get("download_timeout") or self._DEFAULT_CONNECT_TIMEOUT
)
+ deadline = asyncio.get_running_loop().time() + timeout
start_time = time.monotonic()
- async with self._make_request(request, timeout) as response:
- request.meta["download_latency"] = time.monotonic() - start_time
- return await self._read_response(response, request)
-
- async def _read_response(self, response: _ResponseT, request: Request) -> Response:
+ try:
+ async with AsyncExitStack() as stack:
+ async with asyncio.timeout_at(deadline):
+ response = await stack.enter_async_context(
+ self._make_request(request, timeout)
+ )
+ request.meta["download_latency"] = time.monotonic() - start_time
+ return await self._read_response(response, request, deadline)
+ except TimeoutError as e:
+ raise DownloadTimeoutError(
+ f"Getting {request.url} took longer than {timeout} seconds."
+ ) from e
+
+ async def _read_response(
+ self, response: _ResponseT, request: Request, deadline: float
+ ) -> Response:
maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
warnsize = get_warnsize(request.meta, self._default_warnsize)
@@ -215,32 +229,37 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
bytes_received = 0
try:
- async for chunk in self._iter_body_chunks(response):
- response_body.write(chunk)
- bytes_received += len(chunk)
-
- if stop_download := check_stop_download(
- signals.bytes_received, self.crawler, request, data=chunk
- ):
- return make_response(
- **make_response_base_args,
- body=self._get_body(response_body, response, request),
- stop_download=stop_download,
- )
-
- if maxsize and bytes_received > maxsize:
- response_body.truncate(0)
- self._cancel_maxsize(
- bytes_received, maxsize, request, expected=False
- )
-
- if warnsize and bytes_received > warnsize and not reached_warnsize:
- reached_warnsize = True
- logger.warning(
- get_warnsize_msg(
- bytes_received, warnsize, request, expected=False
+ async with asyncio.timeout_at(deadline):
+ async for chunk in self._iter_body_chunks(response):
+ response_body.write(chunk)
+ bytes_received += len(chunk)
+
+ if stop_download := check_stop_download(
+ signals.bytes_received, self.crawler, request, data=chunk
+ ):
+ return make_response(
+ **make_response_base_args,
+ body=self._get_body(response_body, response, request),
+ stop_download=stop_download,
+ )
+
+ if maxsize and bytes_received > maxsize:
+ response_body.truncate(0)
+ self._cancel_maxsize(
+ bytes_received, maxsize, request, expected=False
+ )
+
+ if (
+ warnsize
+ and bytes_received > warnsize
+ and not reached_warnsize
+ ):
+ reached_warnsize = True
+ logger.warning(
+ get_warnsize_msg(
+ bytes_received, warnsize, request, expected=False
+ )
)
- )
except Exception as e:
return self._handle_body_exception(
e,
@@ -264,7 +283,9 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
"""Turn an exception raised while reading the response body into a
partial response, if the request allows for it, or reraise it."""
is_dataloss = self._is_dataloss_exception(exc)
- if not is_dataloss and not self._is_timeout_exception(exc):
+ # TimeoutError also comes from the deadline set in download_request()
+ is_timeout = isinstance(exc, TimeoutError) or self._is_timeout_exception(exc)
+ if not is_dataloss and not is_timeout:
raise exc
fail_on_dataloss: bool = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 9c3e73a33..43e2dced2 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -202,7 +202,9 @@ class HttpxDownloadHandler(_Base):
timeout=timeout,
) as response:
yield response
- except httpx.TimeoutException as e:
+ except httpx.TimeoutException as e: # pragma: no cover
+ # this can only fire with a low-res loop clock, as the timer starts
+ # later than the BaseStreamingDownloadHandler one
raise DownloadTimeoutError(
f"Getting {request.url} took longer than {timeout} seconds."
) from e
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index e7944f6c2..eadcf34bf 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -45,6 +45,7 @@ from .http_resources import (
ResponseHeadersResource,
SetCookie,
Status,
+ Trickle,
UriResource,
put_child,
)
@@ -97,6 +98,7 @@ class Root(BaseResource):
put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))
put_child(self, b"wait", ForeverTakingResource())
put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))
+ put_child(self, b"trickle", Trickle())
put_child(self, b"host", HostHeaderResource())
put_child(self, b"client-ip", ClientIPResource())
put_child(self, b"broken", BrokenDownloadResource())
diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py
index eda524c21..1c728f0cc 100644
--- a/tests/mockserver/http_resources.py
+++ b/tests/mockserver/http_resources.py
@@ -211,6 +211,26 @@ class Delay(LeafResource):
request.finish()
+class Trickle(LeafResource):
+ """Send a body of *n* bytes one byte at a time, one every *interval*
+ seconds."""
+
+ def render_GET(self, request: Request) -> int:
+ n = getarg(request, b"n", 10, type_=int)
+ interval = getarg(request, b"interval", 0.1, type_=float)
+ self._write(request, n, interval)
+ return NOT_DONE_YET
+
+ def _write(self, request: Request, remaining: int, interval: float) -> None:
+ request.write(b"x")
+ if remaining == 1:
+ request.finish()
+ return
+ self.deferRequest(
+ request, interval, self._write, request, remaining - 1, interval
+ )
+
+
class Status(LeafResource):
def render_GET(self, request: Request) -> bytes:
n = getarg(request, b"n", 200, type_=int)
diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.py
index a3fd74fb5..57e98ec6b 100644
--- a/tests/test_downloader_handler_aioftp.py
+++ b/tests/test_downloader_handler_aioftp.py
@@ -231,12 +231,28 @@ async def test_connection_lost() -> None:
await _download(Request(url))
+async def _write_slowly(
+ writer: asyncio.StreamWriter, data: bytes, interval: float
+) -> None:
+ """Write *data* one byte every *interval* seconds, until *writer* is
+ closed."""
+ for i in range(len(data)):
+ if writer.is_closing():
+ return
+ writer.write(data[i : i + 1])
+ await asyncio.sleep(interval)
+
+
def _fake_ftp(
- retr_reply: bytes | None, data: bytes = b""
+ retr_reply: bytes | None, data: bytes = b"", interval: float = 0
) -> Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any]:
"""Return a minimal FTP server connection handler that sends *data*, if
any, and then answers RETR with *retr_reply*, or hangs if *retr_reply* is
- ``None``."""
+ ``None``.
+
+ If *interval* is set, *data* is sent one byte every *interval* seconds,
+ until the client closes the data connection.
+ """
async def handle(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
@@ -267,7 +283,10 @@ def _fake_ftp(
w = await data_writer
if data:
writer.write(b"150 Sending\r\n")
- w.write(data)
+ if interval:
+ await _write_slowly(w, data, interval)
+ else:
+ w.write(data)
if retr_reply is None:
await reader.read()
w.close()
@@ -334,6 +353,15 @@ async def test_timeout_partial_response() -> None:
assert response.flags == ["dataloss"]
+@coroutine_test
+async def test_timeout_slow_transfer() -> None:
+ # the server sends data bytes well within the timeout of each other, but
+ # the whole transfer takes longer than the timeout
+ async with _serve(_fake_ftp(b"226 Done\r\n", b"x" * 20, interval=0.1)) as url:
+ with pytest.raises(DownloadTimeoutError):
+ await _download(Request(url, meta={"download_timeout": 0.5}))
+
+
@coroutine_test
async def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None:
local_filename = str(tmp_path / "file.txt").encode()
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index b151e7bb3..d63717bd8 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -638,6 +638,38 @@ class TestHttpBase(ABC):
assert response.body == b"some bytes"
assert response.flags == ["dataloss"]
+ @coroutine_test
+ async def test_timeout_download_from_spider_slow_body(
+ self, mockserver: MockServer
+ ) -> None:
+ # the server sends body bytes well within the timeout of each other,
+ # but the whole body takes longer than the timeout
+ meta = {"download_timeout": 0.5}
+ request = Request(
+ mockserver.url("/trickle?n=20&interval=0.1", is_secure=self.is_secure),
+ meta=meta,
+ )
+ async with self.get_dh() as download_handler:
+ with pytest.raises(DownloadTimeoutError):
+ await download_handler.download_request(request)
+
+ @coroutine_test
+ async def test_timeout_download_from_spider_slow_body_partial_response(
+ self, mockserver: MockServer
+ ) -> None:
+ if not self.handler_supports_partial_response_on_timeout:
+ pytest.skip("This handler doesn't support partial responses on timeout")
+ meta = {"download_timeout": 0.5, "download_fail_on_dataloss": False}
+ request = Request(
+ mockserver.url("/trickle?n=20&interval=0.1", is_secure=self.is_secure),
+ meta=meta,
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.status == 200
+ assert 0 < len(response.body) < 20
+ assert response.flags == ["dataloss"]
+
@pytest.mark.parametrize("send_header", [True, False])
@coroutine_test
async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: