Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle data loss in HTTP/2 responses (#7989)

Adriancb2f58b4a004
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a67083158..773f58367 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1299,15 +1299,6 @@ Optionally, this can be set on a per-request basis by using the
handler <topics-download-handlers>`, so it's not guaranteed to be supported
by all 3rd-party handlers.
-.. warning::
-
- This setting is ignored by the
- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
- :ref:`download handler <topics-download-handlers>`. In case of a data loss
- error, the corresponding HTTP/2 connection may be corrupted, affecting other
- requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
- failure is always raised for every request that was using that connection.
-
.. setting:: DOWNLOAD_VERIFY_CERTIFICATES
DOWNLOAD_VERIFY_CERTIFICATES
diff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.py
index ab99d68c8..637e418d4 100644
--- a/scrapy/core/_http2/stream.py
+++ b/scrapy/core/_http2/stream.py
@@ -14,7 +14,11 @@ from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
from scrapy import signals
-from scrapy.exceptions import DownloadCancelledError, StopDownload
+from scrapy.exceptions import (
+ DownloadCancelledError,
+ ResponseDataLossError,
+ StopDownload,
+)
from scrapy.http.headers import Headers
from scrapy.utils._download_handlers import (
check_stop_download,
@@ -128,6 +132,10 @@ class Stream:
"download_maxsize", download_maxsize
)
self._download_warnsize = get_warnsize(self._request.meta, download_warnsize)
+ self._fail_on_dataloss: bool = self._request.meta.get(
+ "download_fail_on_dataloss",
+ crawler.settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS"),
+ )
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
@@ -478,7 +486,7 @@ class Stream:
)
elif reason is StreamCloseReason.CONNECTION_LOST:
- self._deferred_response.errback(ResponseFailed(errors))
+ self._handle_connection_lost(errors)
elif reason is StreamCloseReason.INACTIVE:
errors = (InactiveStreamClosed(self._request), *errors)
@@ -494,7 +502,17 @@ class Stream:
)
)
- def _fire_response_deferred(self) -> None:
+ def _handle_connection_lost(self, errors: Sequence[BaseException]) -> None:
+ # A response body is only complete once the stream ends, so having
+ # received the response headers means that the body was cut short.
+ if self._response["status"] is None:
+ self._deferred_response.errback(ResponseFailed(errors))
+ elif self._fail_on_dataloss:
+ self._deferred_response.errback(ResponseDataLossError(str(list(errors))))
+ else:
+ self._fire_response_deferred(flags=["dataloss"])
+
+ def _fire_response_deferred(self, flags: list[str] | None = None) -> None:
"""Builds response from the self._response dict
and fires the response deferred callback with the
generated response instance"""
@@ -505,6 +523,7 @@ class Stream:
status=self._response["status"],
headers=self._response["headers"],
body=self._response["body"].getvalue(),
+ flags=flags,
certificate=self._protocol.metadata["certificate"],
ip_address=self._protocol.metadata["ip_address"],
protocol="h2",
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index a9e29e736..61c67b87e 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -222,6 +222,8 @@ class HttpxDownloadHandler(_Base):
@staticmethod
def _is_dataloss_exception(exc: Exception) -> bool:
+ if HAS_HTTP2 and isinstance(exc, h2.exceptions.InvalidBodyLengthError):
+ return True
return isinstance(
exc, httpx.RemoteProtocolError
) and "peer closed connection without sending complete message body" in str(exc)
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 654183741..5302a7749 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import logging
from time import monotonic
from typing import TYPE_CHECKING
from urllib.parse import urldefrag
@@ -11,9 +12,11 @@ from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import (
DownloadTimeoutError,
NotConfigured,
+ ResponseDataLossError,
UnsupportedURLSchemeError,
)
from scrapy.utils._download_handlers import (
+ get_dataloss_msg,
normalize_bind_address,
wrap_twisted_exceptions,
)
@@ -30,6 +33,9 @@ if TYPE_CHECKING:
from scrapy.spiders import Spider
+logger = logging.getLogger(__name__)
+
+
class H2DownloadHandler(BaseDownloadHandler):
lazy = True
@@ -43,6 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler):
self._pool = H2ConnectionPool(crawler)
self._context_factory = _load_context_factory_from_settings(crawler)
self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
+ self._fail_on_dataloss_warned = False
async def download_request(self, request: Request) -> Response:
if urlparse_cached(request).scheme == "http":
@@ -56,10 +63,16 @@ class H2DownloadHandler(BaseDownloadHandler):
crawler=self._crawler,
)
assert self._crawler.spider
- with wrap_twisted_exceptions():
- return await maybe_deferred_to_future(
- agent.download_request(request, self._crawler.spider)
- )
+ try:
+ with wrap_twisted_exceptions():
+ return await maybe_deferred_to_future(
+ agent.download_request(request, self._crawler.spider)
+ )
+ except ResponseDataLossError:
+ if not self._fail_on_dataloss_warned:
+ logger.warning(get_dataloss_msg(request.url))
+ self._fail_on_dataloss_warned = True
+ raise
async def close(self) -> None:
self._pool.close_connections()
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index 5415274ff..1fc90f33f 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers._httpx import ( # type: ignore[attr-define
HttpxDownloadHandler,
httpx,
)
-from scrapy.exceptions import DownloadFailedError
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import IDNA_REJECTED_HOSTNAMES
@@ -114,7 +113,6 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")
class TestHttp2(TestHttps):
http2 = True
- handler_supports_http2_dataloss = False
default_handler_settings: ClassVar[dict[str, Any]] = {
"HTTPX_HTTP2_ENABLED": True,
@@ -127,13 +125,6 @@ class TestHttp2(TestHttps):
response = await download_handler.download_request(request)
assert response.protocol == "HTTP/2"
- @coroutine_test
- async def test_data_loss_handling(self, mockserver: MockServer) -> None:
- request = Request(mockserver.url("/broken", is_secure=self.is_secure))
- async with self.get_dh() as download_handler:
- with pytest.raises(DownloadFailedError):
- await download_handler.download_request(request)
-
class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
pass
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 37da87ee8..ce65e881e 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -75,7 +75,6 @@ def test_not_configured_without_reactor() -> None:
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
http2 = True
- handler_supports_http2_dataloss = False
@coroutine_test
async def test_protocol(self, mockserver: MockServer) -> None:
@@ -132,13 +131,6 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
with pytest.raises(DownloadFailedError):
await download_handler.download_request(request)
- @coroutine_test
- async def test_data_loss_handling(self, mockserver: MockServer) -> None:
- request = Request(mockserver.url("/broken", is_secure=self.is_secure))
- async with self.get_dh() as download_handler:
- with pytest.raises(DownloadFailedError):
- await download_handler.download_request(request)
-
@coroutine_test
async def test_download_405_data(self, mockserver: MockServer) -> None:
"""Servers without HTTP/2 support answer the connection preface with a
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index aeed0d6d6..3e40292c3 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -22,7 +22,11 @@ from twisted.web.http import Request as TxRequest
from twisted.web.server import NOT_DONE_YET, Site
from twisted.web.static import File
-from scrapy.exceptions import DownloadCancelledError, DownloadTimeoutError
+from scrapy.exceptions import (
+ DownloadCancelledError,
+ DownloadTimeoutError,
+ ResponseDataLossError,
+)
from scrapy.http import JsonRequest, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import (
@@ -267,7 +271,9 @@ class TestHttps2ClientProtocol:
yield client
- if not client.transport.disconnecting:
+ # H2ClientProtocol.connected stays set once the connection is made, so
+ # the underlying transport is what tells whether it is still open.
+ if client.transport.connected:
client.transport.loseConnection()
client.transport.abortConnection()
@@ -512,22 +518,25 @@ class TestHttps2ClientProtocol:
):
await make_request(client, request)
- @inlineCallbacks
- def test_received_dataloss_response(
+ @deferred_f_from_coro_f
+ async def test_received_dataloss_response(
self, server_port: int, client: H2ClientProtocol
- ) -> Generator[Deferred[Any], Any, None]:
- """In case when value of Header Content-Length != len(Received Data)
- ProtocolError is raised"""
- from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415
-
+ ) -> None:
request = Request(url=self.get_url(server_port, "/dataloss"))
- with pytest.raises(ResponseFailed) as exc_info:
- yield make_request_dfd(client, request)
- assert len(exc_info.value.reasons) > 0
- assert any(
- isinstance(error, InvalidBodyLengthError)
- for error in exc_info.value.reasons
+ with pytest.raises(ResponseDataLossError, match=r"InvalidBodyLengthError"):
+ await make_request(client, request)
+
+ @deferred_f_from_coro_f
+ async def test_allow_dataloss_response(
+ self, server_port: int, client: H2ClientProtocol
+ ) -> None:
+ request = Request(
+ url=self.get_url(server_port, "/dataloss"),
+ meta={"download_fail_on_dataloss": False},
)
+ response = await make_request(client, request)
+ assert response.flags == ["dataloss"]
+ assert response.body == Data.DATALOSS
@deferred_f_from_coro_f
async def test_missing_content_length_header(
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index e1158cac8..a5be39934 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -70,13 +70,6 @@ class TestHttpBase(ABC):
http2: bool = False
# whether the handler supports per-request bindaddress
handler_supports_bindaddress_meta: bool = True
- # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
- # stream error (of type PROTOCOL_ERROR) so the client will send
- # RST_STREAM. Some libraries do only this while e.g. h2 also closes the
- # connection (see handling of ProtocolError in
- # h2.connection.H2Connection.receive_data()), thus closing all streams that
- # were using it, and we handle this as a normal exception.
- handler_supports_http2_dataloss: bool = True
# whether the handler sends a request Content-Length header as is, instead
# of building its own (https://github.com/scrapy/scrapy/issues/4919)
handler_supports_custom_content_length: bool = True
@@ -862,8 +855,6 @@ class TestHttpBase(ABC):
@coroutine_test
async def test_download_cause_data_loss(self, mockserver: MockServer) -> None:
- if self.http2 and not self.handler_supports_http2_dataloss:
- pytest.skip("This handler doesn't support dataloss on HTTP/2")
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
with pytest.raises(ResponseDataLossError):
@@ -873,8 +864,6 @@ class TestHttpBase(ABC):
async def test_download_cause_data_loss_double_warning(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
- if self.http2 and not self.handler_supports_http2_dataloss:
- pytest.skip("This handler doesn't support dataloss on HTTP/2")
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
with pytest.raises(ResponseDataLossError):
@@ -890,8 +879,6 @@ class TestHttpBase(ABC):
async def test_download_allow_data_loss_broken(
self, mockserver: MockServer
) -> None:
- if self.http2 and not self.handler_supports_http2_dataloss:
- pytest.skip("This handler doesn't support dataloss on HTTP/2")
request = Request(
mockserver.url("/broken", is_secure=self.is_secure),
meta={"download_fail_on_dataloss": False},
@@ -920,8 +907,6 @@ class TestHttpBase(ABC):
async def test_download_allow_data_loss_via_setting(
self, mockserver: MockServer
) -> None:
- if self.http2 and not self.handler_supports_http2_dataloss:
- pytest.skip("This handler doesn't support dataloss on HTTP/2")
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
async with self.get_dh(
{"DOWNLOAD_FAIL_ON_DATALOSS": False}