Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle data loss in HTTP/2 responses (#7989)
Adriancb2f58b4a004
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex a67083158..773f58367 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1299,15 +1299,6 @@ Optionally, this can be set on a per-request basis by using thehandler <topics-download-handlers>`, so it's not guaranteed to be supportedby all 3rd-party handlers.-.. warning::-- This setting is ignored by the- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`- :ref:`download handler <topics-download-handlers>`. In case of a data loss- error, the corresponding HTTP/2 connection may be corrupted, affecting other- requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``- failure is always raised for every request that was using that connection.-.. setting:: DOWNLOAD_VERIFY_CERTIFICATESDOWNLOAD_VERIFY_CERTIFICATESdiff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.pyindex ab99d68c8..637e418d4 100644--- a/scrapy/core/_http2/stream.py+++ b/scrapy/core/_http2/stream.py@@ -14,7 +14,11 @@ from twisted.python.failure import Failurefrom twisted.web.client import ResponseFailedfrom scrapy import signals-from scrapy.exceptions import DownloadCancelledError, StopDownload+from scrapy.exceptions import (+ DownloadCancelledError,+ ResponseDataLossError,+ StopDownload,+)from scrapy.http.headers import Headersfrom scrapy.utils._download_handlers import (check_stop_download,@@ -128,6 +132,10 @@ class Stream:"download_maxsize", download_maxsize)self._download_warnsize = get_warnsize(self._request.meta, download_warnsize)+ self._fail_on_dataloss: bool = self._request.meta.get(+ "download_fail_on_dataloss",+ crawler.settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS"),+ )# Metadata of an HTTP/2 connection stream# initialized when stream is instantiated@@ -478,7 +486,7 @@ class Stream:)elif reason is StreamCloseReason.CONNECTION_LOST:- self._deferred_response.errback(ResponseFailed(errors))+ self._handle_connection_lost(errors)elif reason is StreamCloseReason.INACTIVE:errors = (InactiveStreamClosed(self._request), *errors)@@ -494,7 +502,17 @@ class Stream:))- def _fire_response_deferred(self) -> None:+ def _handle_connection_lost(self, errors: Sequence[BaseException]) -> None:+ # A response body is only complete once the stream ends, so having+ # received the response headers means that the body was cut short.+ if self._response["status"] is None:+ self._deferred_response.errback(ResponseFailed(errors))+ elif self._fail_on_dataloss:+ self._deferred_response.errback(ResponseDataLossError(str(list(errors))))+ else:+ self._fire_response_deferred(flags=["dataloss"])++ def _fire_response_deferred(self, flags: list[str] | None = None) -> None:"""Builds response from the self._response dictand fires the response deferred callback with thegenerated response instance"""@@ -505,6 +523,7 @@ class Stream:status=self._response["status"],headers=self._response["headers"],body=self._response["body"].getvalue(),+ flags=flags,certificate=self._protocol.metadata["certificate"],ip_address=self._protocol.metadata["ip_address"],protocol="h2",diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex a9e29e736..61c67b87e 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -222,6 +222,8 @@ class HttpxDownloadHandler(_Base):@staticmethoddef _is_dataloss_exception(exc: Exception) -> bool:+ if HAS_HTTP2 and isinstance(exc, h2.exceptions.InvalidBodyLengthError):+ return Truereturn isinstance(exc, httpx.RemoteProtocolError) and "peer closed connection without sending complete message body" in str(exc)diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 654183741..5302a7749 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -1,5 +1,6 @@from __future__ import annotations+import loggingfrom time import monotonicfrom typing import TYPE_CHECKINGfrom urllib.parse import urldefrag@@ -11,9 +12,11 @@ from scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import (DownloadTimeoutError,NotConfigured,+ ResponseDataLossError,UnsupportedURLSchemeError,)from scrapy.utils._download_handlers import (+ get_dataloss_msg,normalize_bind_address,wrap_twisted_exceptions,)@@ -30,6 +33,9 @@ if TYPE_CHECKING:from scrapy.spiders import Spider+logger = logging.getLogger(__name__)++class H2DownloadHandler(BaseDownloadHandler):lazy = True@@ -43,6 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler):self._pool = H2ConnectionPool(crawler)self._context_factory = _load_context_factory_from_settings(crawler)self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")+ self._fail_on_dataloss_warned = Falseasync def download_request(self, request: Request) -> Response:if urlparse_cached(request).scheme == "http":@@ -56,10 +63,16 @@ class H2DownloadHandler(BaseDownloadHandler):crawler=self._crawler,)assert self._crawler.spider- with wrap_twisted_exceptions():- return await maybe_deferred_to_future(- agent.download_request(request, self._crawler.spider)- )+ try:+ with wrap_twisted_exceptions():+ return await maybe_deferred_to_future(+ agent.download_request(request, self._crawler.spider)+ )+ except ResponseDataLossError:+ if not self._fail_on_dataloss_warned:+ logger.warning(get_dataloss_msg(request.url))+ self._fail_on_dataloss_warned = True+ raiseasync def close(self) -> None:self._pool.close_connections()diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 5415274ff..1fc90f33f 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers._httpx import ( # type: ignore[attr-defineHttpxDownloadHandler,httpx,)-from scrapy.exceptions import DownloadFailedErrorfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests import IDNA_REJECTED_HOSTNAMES@@ -114,7 +113,6 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")class TestHttp2(TestHttps):http2 = True- handler_supports_http2_dataloss = Falsedefault_handler_settings: ClassVar[dict[str, Any]] = {"HTTPX_HTTP2_ENABLED": True,@@ -127,13 +125,6 @@ class TestHttp2(TestHttps):response = await download_handler.download_request(request)assert response.protocol == "HTTP/2"- @coroutine_test- async def test_data_loss_handling(self, mockserver: MockServer) -> None:- request = Request(mockserver.url("/broken", is_secure=self.is_secure))- async with self.get_dh() as download_handler:- with pytest.raises(DownloadFailedError):- await download_handler.download_request(request)-class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):passdiff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 37da87ee8..ce65e881e 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -75,7 +75,6 @@ def test_not_configured_without_reactor() -> None:class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):http2 = True- handler_supports_http2_dataloss = False@coroutine_testasync def test_protocol(self, mockserver: MockServer) -> None:@@ -132,13 +131,6 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):with pytest.raises(DownloadFailedError):await download_handler.download_request(request)- @coroutine_test- async def test_data_loss_handling(self, mockserver: MockServer) -> None:- request = Request(mockserver.url("/broken", is_secure=self.is_secure))- async with self.get_dh() as download_handler:- with pytest.raises(DownloadFailedError):- await download_handler.download_request(request)-@coroutine_testasync def test_download_405_data(self, mockserver: MockServer) -> None:"""Servers without HTTP/2 support answer the connection preface with adiff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex aeed0d6d6..3e40292c3 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -22,7 +22,11 @@ from twisted.web.http import Request as TxRequestfrom twisted.web.server import NOT_DONE_YET, Sitefrom twisted.web.static import File-from scrapy.exceptions import DownloadCancelledError, DownloadTimeoutError+from scrapy.exceptions import (+ DownloadCancelledError,+ DownloadTimeoutError,+ ResponseDataLossError,+)from scrapy.http import JsonRequest, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.defer import (@@ -267,7 +271,9 @@ class TestHttps2ClientProtocol:yield client- if not client.transport.disconnecting:+ # H2ClientProtocol.connected stays set once the connection is made, so+ # the underlying transport is what tells whether it is still open.+ if client.transport.connected:client.transport.loseConnection()client.transport.abortConnection()@@ -512,22 +518,25 @@ class TestHttps2ClientProtocol:):await make_request(client, request)- @inlineCallbacks- def test_received_dataloss_response(+ @deferred_f_from_coro_f+ async def test_received_dataloss_response(self, server_port: int, client: H2ClientProtocol- ) -> Generator[Deferred[Any], Any, None]:- """In case when value of Header Content-Length != len(Received Data)- ProtocolError is raised"""- from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415-+ ) -> None:request = Request(url=self.get_url(server_port, "/dataloss"))- with pytest.raises(ResponseFailed) as exc_info:- yield make_request_dfd(client, request)- assert len(exc_info.value.reasons) > 0- assert any(- isinstance(error, InvalidBodyLengthError)- for error in exc_info.value.reasons+ with pytest.raises(ResponseDataLossError, match=r"InvalidBodyLengthError"):+ await make_request(client, request)++ @deferred_f_from_coro_f+ async def test_allow_dataloss_response(+ self, server_port: int, client: H2ClientProtocol+ ) -> None:+ request = Request(+ url=self.get_url(server_port, "/dataloss"),+ meta={"download_fail_on_dataloss": False},)+ response = await make_request(client, request)+ assert response.flags == ["dataloss"]+ assert response.body == Data.DATALOSS@deferred_f_from_coro_fasync def test_missing_content_length_header(diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex e1158cac8..a5be39934 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -70,13 +70,6 @@ class TestHttpBase(ABC):http2: bool = False# whether the handler supports per-request bindaddresshandler_supports_bindaddress_meta: bool = True- # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a- # stream error (of type PROTOCOL_ERROR) so the client will send- # RST_STREAM. Some libraries do only this while e.g. h2 also closes the- # connection (see handling of ProtocolError in- # h2.connection.H2Connection.receive_data()), thus closing all streams that- # were using it, and we handle this as a normal exception.- handler_supports_http2_dataloss: bool = True# whether the handler sends a request Content-Length header as is, instead# of building its own (https://github.com/scrapy/scrapy/issues/4919)handler_supports_custom_content_length: bool = True@@ -862,8 +855,6 @@ class TestHttpBase(ABC):@coroutine_testasync def test_download_cause_data_loss(self, mockserver: MockServer) -> None:- if self.http2 and not self.handler_supports_http2_dataloss:- pytest.skip("This handler doesn't support dataloss on HTTP/2")request = Request(mockserver.url("/broken", is_secure=self.is_secure))async with self.get_dh() as download_handler:with pytest.raises(ResponseDataLossError):@@ -873,8 +864,6 @@ class TestHttpBase(ABC):async def test_download_cause_data_loss_double_warning(self, caplog: pytest.LogCaptureFixture, mockserver: MockServer) -> None:- if self.http2 and not self.handler_supports_http2_dataloss:- pytest.skip("This handler doesn't support dataloss on HTTP/2")request = Request(mockserver.url("/broken", is_secure=self.is_secure))async with self.get_dh() as download_handler:with pytest.raises(ResponseDataLossError):@@ -890,8 +879,6 @@ class TestHttpBase(ABC):async def test_download_allow_data_loss_broken(self, mockserver: MockServer) -> None:- if self.http2 and not self.handler_supports_http2_dataloss:- pytest.skip("This handler doesn't support dataloss on HTTP/2")request = Request(mockserver.url("/broken", is_secure=self.is_secure),meta={"download_fail_on_dataloss": False},@@ -920,8 +907,6 @@ class TestHttpBase(ABC):async def test_download_allow_data_loss_via_setting(self, mockserver: MockServer) -> None:- if self.http2 and not self.handler_supports_http2_dataloss:- pytest.skip("This handler doesn't support dataloss on HTTP/2")request = Request(mockserver.url("/broken", is_secure=self.is_secure))async with self.get_dh({"DOWNLOAD_FAIL_ON_DATALOSS": False}