Scrapy, a fast high-level web crawling & scraping framework for Python.
Return partial responses on timeout when fail_on_dataloss is False (#8084)
Adriandac85847541a
diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.pyindex e365123fc..5adbf660e 100644--- a/scrapy/core/downloader/handlers/_aioftp.py+++ b/scrapy/core/downloader/handlers/_aioftp.py@@ -269,6 +269,10 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):def _is_dataloss_exception(exc: Exception) -> bool:return isinstance(exc, aioftp.StatusCodeError)+ @staticmethod+ def _is_timeout_exception(exc: Exception) -> bool:+ return isinstance(exc, TimeoutError)+def _log_tls_info(self, response: _AioftpResponse, request: Request) -> None:if ssl_object := response.control.writer.get_extra_info("ssl_object"):_log_sslobj_debug_info(ssl_object)diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.pyindex 566575568..3317ffbbe 100644--- a/scrapy/core/downloader/handlers/_aiohttp.py+++ b/scrapy/core/downloader/handlers/_aiohttp.py@@ -157,5 +157,9 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):def _is_dataloss_exception(exc: Exception) -> bool:return isinstance(exc, aiohttp.ClientPayloadError)+ @staticmethod+ def _is_timeout_exception(exc: Exception) -> bool:+ return isinstance(exc, TimeoutError)+async def close(self) -> None:await self._session.close()diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.pyindex 1888e7a26..678ab9f22 100644--- a/scrapy/core/downloader/handlers/_base_streaming.py+++ b/scrapy/core/downloader/handlers/_base_streaming.py@@ -135,6 +135,13 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon"""Return True if ``exc`` represents dataloss."""raise NotImplementedError+ @staticmethod+ @abstractmethod+ def _is_timeout_exception(exc: Exception) -> bool:+ """Return True if ``exc`` represents the download timeout being hit+ while reading the response body."""+ raise NotImplementedError+@staticmethoddef _check_deps_installed() -> None:"""Raise NotConfigured if the required deps are not installed."""@@ -237,27 +244,45 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon))except Exception as e:- if not self._is_dataloss_exception(e):- raise- fail_on_dataloss: bool = request.meta.get(- "download_fail_on_dataloss", self._fail_on_dataloss+ return self._handle_body_exception(+ e,+ request,+ make_response_base_args,+ self._get_body(response_body, response, request),)- if not fail_on_dataloss:- return make_response(- **make_response_base_args,- body=self._get_body(response_body, response, request),- flags=["dataloss"],- )- if not self._fail_on_dataloss_warned:- logger.warning(get_dataloss_msg(request.url))- self._fail_on_dataloss_warned = True- raise ResponseDataLossError(str(e)) from ereturn make_response(**make_response_base_args,body=self._get_body(response_body, response, request),)+ def _handle_body_exception(+ self,+ exc: Exception,+ request: Request,+ make_response_base_args: _BaseResponseArgs,+ body: bytes,+ ) -> Response:+ """Turn an exception raised while reading the response body into a+ partial response, if the request allows for it, or reraise it."""+ is_dataloss = self._is_dataloss_exception(exc)+ if not is_dataloss and not self._is_timeout_exception(exc):+ raise exc+ fail_on_dataloss: bool = request.meta.get(+ "download_fail_on_dataloss", self._fail_on_dataloss+ )+ if not fail_on_dataloss:+ return make_response(+ **make_response_base_args, body=body, flags=["dataloss"]+ )+ if not is_dataloss:+ # let the timeout be reported as usual+ raise exc+ if not self._fail_on_dataloss_warned:+ logger.warning(get_dataloss_msg(request.url))+ self._fail_on_dataloss_warned = True+ raise ResponseDataLossError(str(exc)) from exc+@staticmethoddef _request_headers(request: Request) -> Headers:"""Get a prepared copy of the request headers.diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 61c67b87e..429b32cc5 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -228,6 +228,10 @@ class HttpxDownloadHandler(_Base):exc, httpx.RemoteProtocolError) and "peer closed connection without sending complete message body" in str(exc)+ @staticmethod+ def _is_timeout_exception(exc: Exception) -> bool:+ return isinstance(exc, httpx.ReadTimeout)+def _log_tls_info(self, response: httpx.Response, request: Request) -> None:network_stream: AsyncNetworkStream = response.extensions["network_stream"]extra_ssl_object = network_stream.get_extra_info("ssl_object")diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex d00836fa7..f4f09127b 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -533,10 +533,21 @@ class _ScrapyAgent:d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)# check download timeout- self._timeout_cl = reactor.callLater(timeout, d3.cancel)+ self._timeout_cl = reactor.callLater(timeout, self._cb_timed_out, d3, request)d3.addBoth(self._cb_timeout, request, url, timeout)return d3+ def _cb_timed_out(self, d3: Deferred[Response], request: Request) -> None:+ fail_on_dataloss = request.meta.get(+ "download_fail_on_dataloss", self._fail_on_dataloss+ )+ if not fail_on_dataloss and self._txresponse is not None:+ # let the response be delivered as a partial one instead of+ # cancelling the request outright+ self._txresponse._transport.stopProducing()+ else:+ d3.cancel()+def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:if self._timeout_cl.active():self._timeout_cl.cancel()@@ -546,6 +557,9 @@ class _ScrapyAgent:if self._txresponse:self._txresponse._transport.stopProducing()+ if not isinstance(result, Failure):+ return result+raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.pyindex bf98920f7..610c76964 100644--- a/tests/test_downloader_handler_aioftp.py+++ b/tests/test_downloader_handler_aioftp.py@@ -232,10 +232,11 @@ async def test_connection_lost() -> None:def _fake_ftp(- retr_reply: bytes, data: bytes = b""+ retr_reply: bytes | None, data: bytes = b"") -> Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any]:"""Return a minimal FTP server connection handler that sends *data*, if- any, and then answers RETR with *retr_reply*."""+ any, and then answers RETR with *retr_reply*, or hangs if *retr_reply* is+ ``None``."""async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter@@ -267,8 +268,12 @@ def _fake_ftp(if data:writer.write(b"150 Sending\r\n")w.write(data)+ if retr_reply is None:+ await reader.read()w.close()data_server.close()+ if retr_reply is None:+ breakwriter.write(retr_reply)await writer.drain()writer.close()@@ -286,6 +291,7 @@ async def _serve(yield f"ftp://127.0.0.1:{port}/file.txt"finally:server.close()+ await server.wait_closed()@coroutine_test@@ -315,6 +321,19 @@ async def test_dataloss_allowed() -> None:assert response.flags == ["dataloss"]+@coroutine_test+async def test_timeout_partial_response() -> None:+ async with _serve(_fake_ftp(None, data=b"partial")) as url:+ response = await _download(+ Request(+ url,+ meta={"download_timeout": 0.5, "download_fail_on_dataloss": False},+ )+ )+ assert response.body == b"partial"+ assert response.flags == ["dataloss"]++@coroutine_testasync def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None:local_filename = str(tmp_path / "file.txt").encode()diff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.pyindex 72b0811d5..02ef5fe4f 100644--- a/tests/test_downloader_handler_aiohttp.py+++ b/tests/test_downloader_handler_aiohttp.py@@ -28,6 +28,8 @@ pytestmark = pytest.mark.only_asyncioclass AiohttpDownloadHandlerMixin:+ handler_supports_partial_response_on_timeout = True+@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:return AiohttpDownloadHandlerdiff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 1fc90f33f..72d2ce834 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -62,6 +62,8 @@ else:class HttpxDownloadHandlerMixin:+ handler_supports_partial_response_on_timeout = True+@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:return HttpxDownloadHandlerdiff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex 2c08e6eb0..9e58e3105 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -56,6 +56,8 @@ pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a reclass HTTP11DownloadHandlerMixin:+ handler_supports_partial_response_on_timeout = True+@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:return HTTP11DownloadHandlerdiff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex a5be39934..985a88276 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -70,6 +70,9 @@ class TestHttpBase(ABC):http2: bool = False# whether the handler supports per-request bindaddresshandler_supports_bindaddress_meta: bool = True+ # whether the handler can return the body received so far, instead of+ # erroring out, when a request times out mid-response+ handler_supports_partial_response_on_timeout: bool = False# whether the handler sends a request Content-Length header as is, instead# of building its own (https://github.com/scrapy/scrapy/issues/4919)handler_supports_custom_content_length: bool = True@@ -435,6 +438,25 @@ class TestHttpBase(ABC):with pytest.raises(DownloadTimeoutError):await maybe_deferred_to_future(d)+ @coroutine_test+ async def test_timeout_download_from_spider_server_hangs_partial_response(+ self, mockserver: MockServer+ ) -> None:+ if not self.handler_supports_partial_response_on_timeout:+ pytest.skip("This handler doesn't support partial responses on timeout")+ # client connects, server sends headers and some body bytes but hangs;+ # with dataloss failures disabled the body received so far is+ # returned instead of raising a timeout error+ meta = {"download_timeout": 0.5, "download_fail_on_dataloss": False}+ request = Request(+ mockserver.url("/hang-after-headers", is_secure=self.is_secure), meta=meta+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.status == 200+ assert response.body == b"some bytes"+ assert response.flags == ["dataloss"]+@pytest.mark.parametrize("send_header", [True, False])@coroutine_testasync def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: