Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Return partial responses on timeout when fail_on_dataloss is False (#8084)

Adriandac85847541a
diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.py
index e365123fc..5adbf660e 100644
--- a/scrapy/core/downloader/handlers/_aioftp.py
+++ b/scrapy/core/downloader/handlers/_aioftp.py
@@ -269,6 +269,10 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):
def _is_dataloss_exception(exc: Exception) -> bool:
return isinstance(exc, aioftp.StatusCodeError)
+ @staticmethod
+ def _is_timeout_exception(exc: Exception) -> bool:
+ return isinstance(exc, TimeoutError)
+
def _log_tls_info(self, response: _AioftpResponse, request: Request) -> None:
if ssl_object := response.control.writer.get_extra_info("ssl_object"):
_log_sslobj_debug_info(ssl_object)
diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.py
index 566575568..3317ffbbe 100644
--- a/scrapy/core/downloader/handlers/_aiohttp.py
+++ b/scrapy/core/downloader/handlers/_aiohttp.py
@@ -157,5 +157,9 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):
def _is_dataloss_exception(exc: Exception) -> bool:
return isinstance(exc, aiohttp.ClientPayloadError)
+ @staticmethod
+ def _is_timeout_exception(exc: Exception) -> bool:
+ return isinstance(exc, TimeoutError)
+
async def close(self) -> None:
await self._session.close()
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 1888e7a26..678ab9f22 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -135,6 +135,13 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
"""Return True if ``exc`` represents dataloss."""
raise NotImplementedError
+ @staticmethod
+ @abstractmethod
+ def _is_timeout_exception(exc: Exception) -> bool:
+ """Return True if ``exc`` represents the download timeout being hit
+ while reading the response body."""
+ raise NotImplementedError
+
@staticmethod
def _check_deps_installed() -> None:
"""Raise NotConfigured if the required deps are not installed."""
@@ -237,27 +244,45 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
)
)
except Exception as e:
- if not self._is_dataloss_exception(e):
- raise
- fail_on_dataloss: bool = request.meta.get(
- "download_fail_on_dataloss", self._fail_on_dataloss
+ return self._handle_body_exception(
+ e,
+ request,
+ make_response_base_args,
+ self._get_body(response_body, response, request),
)
- if not fail_on_dataloss:
- return make_response(
- **make_response_base_args,
- body=self._get_body(response_body, response, request),
- flags=["dataloss"],
- )
- if not self._fail_on_dataloss_warned:
- logger.warning(get_dataloss_msg(request.url))
- self._fail_on_dataloss_warned = True
- raise ResponseDataLossError(str(e)) from e
return make_response(
**make_response_base_args,
body=self._get_body(response_body, response, request),
)
+ def _handle_body_exception(
+ self,
+ exc: Exception,
+ request: Request,
+ make_response_base_args: _BaseResponseArgs,
+ body: bytes,
+ ) -> Response:
+ """Turn an exception raised while reading the response body into a
+ partial response, if the request allows for it, or reraise it."""
+ is_dataloss = self._is_dataloss_exception(exc)
+ if not is_dataloss and not self._is_timeout_exception(exc):
+ raise exc
+ fail_on_dataloss: bool = request.meta.get(
+ "download_fail_on_dataloss", self._fail_on_dataloss
+ )
+ if not fail_on_dataloss:
+ return make_response(
+ **make_response_base_args, body=body, flags=["dataloss"]
+ )
+ if not is_dataloss:
+ # let the timeout be reported as usual
+ raise exc
+ if not self._fail_on_dataloss_warned:
+ logger.warning(get_dataloss_msg(request.url))
+ self._fail_on_dataloss_warned = True
+ raise ResponseDataLossError(str(exc)) from exc
+
@staticmethod
def _request_headers(request: Request) -> Headers:
"""Get a prepared copy of the request headers.
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 61c67b87e..429b32cc5 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -228,6 +228,10 @@ class HttpxDownloadHandler(_Base):
exc, httpx.RemoteProtocolError
) and "peer closed connection without sending complete message body" in str(exc)
+ @staticmethod
+ def _is_timeout_exception(exc: Exception) -> bool:
+ return isinstance(exc, httpx.ReadTimeout)
+
def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
network_stream: AsyncNetworkStream = response.extensions["network_stream"]
extra_ssl_object = network_stream.get_extra_info("ssl_object")
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index d00836fa7..f4f09127b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -533,10 +533,21 @@ class _ScrapyAgent:
d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)
# check download timeout
- self._timeout_cl = reactor.callLater(timeout, d3.cancel)
+ self._timeout_cl = reactor.callLater(timeout, self._cb_timed_out, d3, request)
d3.addBoth(self._cb_timeout, request, url, timeout)
return d3
+ def _cb_timed_out(self, d3: Deferred[Response], request: Request) -> None:
+ fail_on_dataloss = request.meta.get(
+ "download_fail_on_dataloss", self._fail_on_dataloss
+ )
+ if not fail_on_dataloss and self._txresponse is not None:
+ # let the response be delivered as a partial one instead of
+ # cancelling the request outright
+ self._txresponse._transport.stopProducing()
+ else:
+ d3.cancel()
+
def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
if self._timeout_cl.active():
self._timeout_cl.cancel()
@@ -546,6 +557,9 @@ class _ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
+ if not isinstance(result, Failure):
+ return result
+
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
diff --git a/tests/test_downloader_handler_aioftp.py b/tests/test_downloader_handler_aioftp.py
index bf98920f7..610c76964 100644
--- a/tests/test_downloader_handler_aioftp.py
+++ b/tests/test_downloader_handler_aioftp.py
@@ -232,10 +232,11 @@ async def test_connection_lost() -> None:
def _fake_ftp(
- retr_reply: bytes, data: bytes = b""
+ retr_reply: bytes | None, data: bytes = b""
) -> Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any]:
"""Return a minimal FTP server connection handler that sends *data*, if
- any, and then answers RETR with *retr_reply*."""
+ any, and then answers RETR with *retr_reply*, or hangs if *retr_reply* is
+ ``None``."""
async def handle(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
@@ -267,8 +268,12 @@ def _fake_ftp(
if data:
writer.write(b"150 Sending\r\n")
w.write(data)
+ if retr_reply is None:
+ await reader.read()
w.close()
data_server.close()
+ if retr_reply is None:
+ break
writer.write(retr_reply)
await writer.drain()
writer.close()
@@ -286,6 +291,7 @@ async def _serve(
yield f"ftp://127.0.0.1:{port}/file.txt"
finally:
server.close()
+ await server.wait_closed()
@coroutine_test
@@ -315,6 +321,19 @@ async def test_dataloss_allowed() -> None:
assert response.flags == ["dataloss"]
+@coroutine_test
+async def test_timeout_partial_response() -> None:
+ async with _serve(_fake_ftp(None, data=b"partial")) as url:
+ response = await _download(
+ Request(
+ url,
+ meta={"download_timeout": 0.5, "download_fail_on_dataloss": False},
+ )
+ )
+ assert response.body == b"partial"
+ assert response.flags == ["dataloss"]
+
+
@coroutine_test
async def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None:
local_filename = str(tmp_path / "file.txt").encode()
diff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.py
index 72b0811d5..02ef5fe4f 100644
--- a/tests/test_downloader_handler_aiohttp.py
+++ b/tests/test_downloader_handler_aiohttp.py
@@ -28,6 +28,8 @@ pytestmark = pytest.mark.only_asyncio
class AiohttpDownloadHandlerMixin:
+ handler_supports_partial_response_on_timeout = True
+
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return AiohttpDownloadHandler
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index 1fc90f33f..72d2ce834 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -62,6 +62,8 @@ else:
class HttpxDownloadHandlerMixin:
+ handler_supports_partial_response_on_timeout = True
+
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return HttpxDownloadHandler
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 2c08e6eb0..9e58e3105 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -56,6 +56,8 @@ pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a re
class HTTP11DownloadHandlerMixin:
+ handler_supports_partial_response_on_timeout = True
+
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return HTTP11DownloadHandler
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index a5be39934..985a88276 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -70,6 +70,9 @@ class TestHttpBase(ABC):
http2: bool = False
# whether the handler supports per-request bindaddress
handler_supports_bindaddress_meta: bool = True
+ # whether the handler can return the body received so far, instead of
+ # erroring out, when a request times out mid-response
+ handler_supports_partial_response_on_timeout: bool = False
# whether the handler sends a request Content-Length header as is, instead
# of building its own (https://github.com/scrapy/scrapy/issues/4919)
handler_supports_custom_content_length: bool = True
@@ -435,6 +438,25 @@ class TestHttpBase(ABC):
with pytest.raises(DownloadTimeoutError):
await maybe_deferred_to_future(d)
+ @coroutine_test
+ async def test_timeout_download_from_spider_server_hangs_partial_response(
+ self, mockserver: MockServer
+ ) -> None:
+ if not self.handler_supports_partial_response_on_timeout:
+ pytest.skip("This handler doesn't support partial responses on timeout")
+ # client connects, server sends headers and some body bytes but hangs;
+ # with dataloss failures disabled the body received so far is
+ # returned instead of raising a timeout error
+ meta = {"download_timeout": 0.5, "download_fail_on_dataloss": False}
+ request = Request(
+ mockserver.url("/hang-after-headers", is_secure=self.is_secure), meta=meta
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.status == 200
+ assert response.body == b"some bytes"
+ assert response.flags == ["dataloss"]
+
@pytest.mark.parametrize("send_header", [True, False])
@coroutine_test
async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: