Scrapy, a fast high-level web crawling & scraping framework for Python.
Force download timeouts in BaseStreamingDownloadHandler. (#8481)
Andrey Rakhmatullinaeaec6a5bd84
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 8465d4697..bf663a05f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1248,9 +1248,12 @@ Default: ``180``The amount of time (in secs) that the downloader will wait before timing out.+A value of ``0`` or ``None`` doesn't mean a timeout of 0 or infinity: a+non-zero handler-specific value will be used instead.+.. note::- This timeout can be per-request using the :reqmeta:`download_timeout`+ This timeout can be set per-request using the :reqmeta:`download_timeout`:attr:`.Request.meta` key... note::diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.pyindex ea4b661bb..3903efcb6 100644--- a/scrapy/core/downloader/handlers/_aioftp.py+++ b/scrapy/core/downloader/handlers/_aioftp.py@@ -218,9 +218,9 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):return super()._get_body(buffer, response, request)async def _read_response(- self, response: _AioftpResponse, request: Request+ self, response: _AioftpResponse, request: Request, deadline: float) -> Response:- result = await super()._read_response(response, request)+ result = await super()._read_response(response, request, deadline)if response.data is not None:self._add_ftp_headers(result, response, request)return resultdiff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.pyindex 615ec4391..799126ad0 100644--- a/scrapy/core/downloader/handlers/_base_streaming.py+++ b/scrapy/core/downloader/handlers/_base_streaming.py@@ -1,9 +1,11 @@from __future__ import annotations+import asyncioimport base64import loggingimport timefrom abc import abstractmethod+from contextlib import AsyncExitStackfrom io import BytesIOfrom pathlib import Pathfrom typing import (@@ -24,6 +26,7 @@ from w3lib.url import add_http_if_no_schemefrom scrapy import Request, signalsfrom scrapy.exceptions import (DownloadCancelledError,+ DownloadTimeoutError,NotConfigured,ResponseDataLossError,)@@ -72,13 +75,12 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])_DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10experimental: ClassVar[bool] = False- requires_asyncio: ClassVar[bool] = True# require subclasses to disable proxies explicitly with an explanationsupports_proxies: ClassVar[bool] = Truesupports_per_request_bindaddress: ClassVar[bool] = Falsedef __init__(self, crawler: Crawler):- if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover+ if not is_asyncio_available(): # pragma: no coverraise NotConfigured(f"{type(self).__name__} requires the asyncio support.")self._check_deps_installed()super().__init__(crawler)@@ -167,15 +169,27 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])f"The 'bindaddress' request meta key is not supported by"f" {type(self).__name__} and will be ignored.")- timeout: float = request.meta.get(- "download_timeout", self._DEFAULT_CONNECT_TIMEOUT+ timeout: float = (+ request.meta.get("download_timeout") or self._DEFAULT_CONNECT_TIMEOUT)+ deadline = asyncio.get_running_loop().time() + timeoutstart_time = time.monotonic()- async with self._make_request(request, timeout) as response:- request.meta["download_latency"] = time.monotonic() - start_time- return await self._read_response(response, request)-- async def _read_response(self, response: _ResponseT, request: Request) -> Response:+ try:+ async with AsyncExitStack() as stack:+ async with asyncio.timeout_at(deadline):+ response = await stack.enter_async_context(+ self._make_request(request, timeout)+ )+ request.meta["download_latency"] = time.monotonic() - start_time+ return await self._read_response(response, request, deadline)+ except TimeoutError as e:+ raise DownloadTimeoutError(+ f"Getting {request.url} took longer than {timeout} seconds."+ ) from e++ async def _read_response(+ self, response: _ResponseT, request: Request, deadline: float+ ) -> Response:maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)warnsize = get_warnsize(request.meta, self._default_warnsize)@@ -215,32 +229,37 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])bytes_received = 0try:- async for chunk in self._iter_body_chunks(response):- response_body.write(chunk)- bytes_received += len(chunk)-- if stop_download := check_stop_download(- signals.bytes_received, self.crawler, request, data=chunk- ):- return make_response(- **make_response_base_args,- body=self._get_body(response_body, response, request),- stop_download=stop_download,- )-- if maxsize and bytes_received > maxsize:- response_body.truncate(0)- self._cancel_maxsize(- bytes_received, maxsize, request, expected=False- )-- if warnsize and bytes_received > warnsize and not reached_warnsize:- reached_warnsize = True- logger.warning(- get_warnsize_msg(- bytes_received, warnsize, request, expected=False+ async with asyncio.timeout_at(deadline):+ async for chunk in self._iter_body_chunks(response):+ response_body.write(chunk)+ bytes_received += len(chunk)++ if stop_download := check_stop_download(+ signals.bytes_received, self.crawler, request, data=chunk+ ):+ return make_response(+ **make_response_base_args,+ body=self._get_body(response_body, response, request),+ stop_download=stop_download,+ )++ if maxsize and bytes_received > maxsize:+ response_body.truncate(0)+ self._cancel_maxsize(+ bytes_received, maxsize, request, expected=False+ )++ if (+ warnsize+ and bytes_received > warnsize+ and not reached_warnsize+ ):+ reached_warnsize = True+ logger.warning(+ get_warnsize_msg(+ bytes_received, warnsize, request, expected=False+ ))- )except Exception as e:return self._handle_body_exception(e,@@ -264,7 +283,9 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])"""Turn an exception raised while reading the response body into apartial response, if the request allows for it, or reraise it."""is_dataloss = self._is_dataloss_exception(exc)- if not is_dataloss and not self._is_timeout_exception(exc):+ # TimeoutError also comes from the deadline set in download_request()+ is_timeout = isinstance(exc, TimeoutError) or self._is_timeout_exception(exc)+ if not is_dataloss and not is_timeout:raise excfail_on_dataloss: bool = request.meta.get("download_fail_on_dataloss", self._fail_on_datalossdiff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 9c3e73a33..43e2dced2 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -202,7 +202,9 @@ class HttpxDownloadHandler(_Base):timeout=timeout,) as response:yield response- except httpx.TimeoutException as e:+ except httpx.TimeoutException as e: # pragma: no cover+ # this can only fire with a low-res loop clock, as the timer starts+ # later than the BaseStreamingDownloadHandler oneraise DownloadTimeoutError(f"Getting {request.url} took longer than {timeout} seconds.") from ediff --git a/tests/mockserver/http.py b/tests/mockserver/http.pyindex e7944f6c2..eadcf34bf 100644--- a/tests/mockserver/http.py+++ b/tests/mockserver/http.py@@ -45,6 +45,7 @@ from .http_resources import (ResponseHeadersResource,SetCookie,Status,+ Trickle,UriResource,put_child,)@@ -97,6 +98,7 @@ class Root(BaseResource):put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))put_child(self, b"wait", ForeverTakingResource())put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))+ put_child(self, b"trickle", Trickle())put_child(self, b"host", HostHeaderResource())put_child(self, b"client-ip", ClientIPResource())put_child(self, b"broken", BrokenDownloadResource())diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.pyindex eda524c21..1c728f0cc 100644--- a/tests/mockserver/http_resources.py+++ b/tests/mockserver/http_resources.py@@ -211,6 +211,26 @@ class Delay(LeafResource):request.finish()+class Trickle(LeafResource):+ """Send a body of *n* bytes one byte at a time, one every *interval*+ seconds."""++ def render_GET(self, request: Request) -> int:+ n = getarg(request, b"n", 10, type_=int)+ interval = getarg(request, b"interval", 0.1, type_=float)+ self._write(request, n, interval)+ return NOT_DONE_YET++ def _write(self, request: Request, remaining: int, interval: float) -> None:+ request.write(b"x")+ if remaining == 1:+ request.finish()+ return+ self.deferRequest(+ request, interval, self._write, request, remaining - 1, interval+ )++class Status(LeafResource):def render_GET(self, request: Request) -> bytes:n = getarg(request, b"n", 200, type_=int)diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.pyindex a3fd74fb5..57e98ec6b 100644--- a/tests/test_downloader_handler_aioftp.py+++ b/tests/test_downloader_handler_aioftp.py@@ -231,12 +231,28 @@ async def test_connection_lost() -> None:await _download(Request(url))+async def _write_slowly(+ writer: asyncio.StreamWriter, data: bytes, interval: float+) -> None:+ """Write *data* one byte every *interval* seconds, until *writer* is+ closed."""+ for i in range(len(data)):+ if writer.is_closing():+ return+ writer.write(data[i : i + 1])+ await asyncio.sleep(interval)++def _fake_ftp(- retr_reply: bytes | None, data: bytes = b""+ retr_reply: bytes | None, data: bytes = b"", interval: float = 0) -> Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any]:"""Return a minimal FTP server connection handler that sends *data*, ifany, and then answers RETR with *retr_reply*, or hangs if *retr_reply* is- ``None``."""+ ``None``.++ If *interval* is set, *data* is sent one byte every *interval* seconds,+ until the client closes the data connection.+ """async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter@@ -267,7 +283,10 @@ def _fake_ftp(w = await data_writerif data:writer.write(b"150 Sending\r\n")- w.write(data)+ if interval:+ await _write_slowly(w, data, interval)+ else:+ w.write(data)if retr_reply is None:await reader.read()w.close()@@ -334,6 +353,15 @@ async def test_timeout_partial_response() -> None:assert response.flags == ["dataloss"]+@coroutine_test+async def test_timeout_slow_transfer() -> None:+ # the server sends data bytes well within the timeout of each other, but+ # the whole transfer takes longer than the timeout+ async with _serve(_fake_ftp(b"226 Done\r\n", b"x" * 20, interval=0.1)) as url:+ with pytest.raises(DownloadTimeoutError):+ await _download(Request(url, meta={"download_timeout": 0.5}))++@coroutine_testasync def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None:local_filename = str(tmp_path / "file.txt").encode()diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex b151e7bb3..d63717bd8 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -638,6 +638,38 @@ class TestHttpBase(ABC):assert response.body == b"some bytes"assert response.flags == ["dataloss"]+ @coroutine_test+ async def test_timeout_download_from_spider_slow_body(+ self, mockserver: MockServer+ ) -> None:+ # the server sends body bytes well within the timeout of each other,+ # but the whole body takes longer than the timeout+ meta = {"download_timeout": 0.5}+ request = Request(+ mockserver.url("/trickle?n=20&interval=0.1", is_secure=self.is_secure),+ meta=meta,+ )+ async with self.get_dh() as download_handler:+ with pytest.raises(DownloadTimeoutError):+ await download_handler.download_request(request)++ @coroutine_test+ async def test_timeout_download_from_spider_slow_body_partial_response(+ self, mockserver: MockServer+ ) -> None:+ if not self.handler_supports_partial_response_on_timeout:+ pytest.skip("This handler doesn't support partial responses on timeout")+ meta = {"download_timeout": 0.5, "download_fail_on_dataloss": False}+ request = Request(+ mockserver.url("/trickle?n=20&interval=0.1", is_secure=self.is_secure),+ meta=meta,+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.status == 200+ assert 0 < len(response.body) < 20+ assert response.flags == ["dataloss"]+@pytest.mark.parametrize("send_header", [True, False])@coroutine_testasync def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: