Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a HTTP2_MAX_FRAME_SIZE setting (#7988)
Adriana22523bfd3d4
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex d95e7b52b..2d04b621c 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -209,12 +209,8 @@ Known limitations of the HTTP/2 support:- No support for HTTP/2 Cleartext (h2c), since no major browser supportsHTTP/2 unencrypted (refer `http2 faq`_).-- No setting to specify a maximum `frame size`_ larger than the default- value, 16384. Connections to servers that send a larger frame will fail.-- No support for `server pushes`_, which are ignored.-.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 010c9d1c6..e58815fad 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1499,6 +1499,28 @@ Default: ``None``The Project ID that will be used when storing data on `Google Cloud Storage`_.+.. setting:: HTTP2_MAX_FRAME_SIZE++HTTP2_MAX_FRAME_SIZE+--------------------++.. versionadded:: VERSION++Default: ``16384``++Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and+``16777215``. Connections to servers that send a larger frame fail.++Raise it for servers that send larger frames regardless of this value. Note+that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked+once per received frame, so a higher value allows a response to exceed them by+more before being caught.++:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores+this setting, as ``httpx`` does not allow configuring the frame size.++.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2+.. setting:: ITEM_PIPELINESITEM_PIPELINESdiff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.pyindex aff985d94..1e673ce8b 100644--- a/scrapy/core/_http2/protocol.py+++ b/scrapy/core/_http2/protocol.py@@ -21,6 +21,7 @@ from h2.events import (WindowUpdated,)from h2.exceptions import FrameTooLargeError, H2Error+from h2.settings import SettingCodesfrom twisted.internet.interfaces import (IAddress,IHandshakeListener,@@ -261,6 +262,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):# Initiate H2 Connectionself.conn.initiate_connection()+ max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE")+ if max_frame_size != self.conn.local_settings.max_frame_size:+ self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size})self._write_to_transport()def _lose_connection_with_error(self, errors: list[BaseException]) -> None:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d7b30b849..b97844103 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -108,6 +108,7 @@ __all__ = ["FTP_PASSWORD","FTP_USER","GCS_PROJECT_ID",+ "HTTP2_MAX_FRAME_SIZE","HTTPAUTH_DOMAIN","HTTPAUTH_PASS","HTTPAUTH_USER",@@ -402,6 +403,8 @@ FTP_PASSWORD = "guest" # noqa: S105GCS_PROJECT_ID = None+HTTP2_MAX_FRAME_SIZE = 16384+HTTPAUTH_USER = ""HTTPAUTH_PASS = ""HTTPAUTH_DOMAIN = Nonediff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 161015231..075f1acb9 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -37,6 +37,7 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generatorfrom scrapy.core._http2.protocol import H2ClientProtocol+ from scrapy.crawler import Crawlerpytestmark = [@@ -236,9 +237,16 @@ class TestHttps2ClientProtocol:) + self.certificate_file.read_text(encoding="utf-8")return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return]+ @pytest.fixture+ def crawler(self, request: pytest.FixtureRequest) -> Crawler:+ return get_crawler(settings_dict=getattr(request, "param", None))+@async_yield_fixture # type: ignore[untyped-decorator]async def client(- self, server_port: int, client_certificate: PrivateCertificate+ self,+ server_port: int,+ client_certificate: PrivateCertificate,+ crawler: Crawler,) -> AsyncGenerator[H2ClientProtocol]:from twisted.internet import reactor@@ -250,7 +258,7 @@ class TestHttps2ClientProtocol:acceptableProtocols=[b"h2"],)uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8"))- h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred())+ h2_client_factory = H2ClientFactory(uri, crawler, Deferred())client_endpoint = SSL4ClientEndpoint(reactor, self.host, server_port, client_options)@@ -312,6 +320,17 @@ class TestHttps2ClientProtocol:request = Request(self.get_url(server_port, "/get-data-html-large"))await self._check_GET(client, request, Data.HTML_LARGE, 200)+ @pytest.mark.parametrize(+ "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True+ )+ @deferred_f_from_coro_f+ async def test_GET_large_frames(+ self, server_port: int, client: H2ClientProtocol+ ) -> None:+ request = Request(self.get_url(server_port, "/get-data-html-large"))+ await self._check_GET(client, request, Data.HTML_LARGE, 200)+ assert client.conn.local_settings.max_frame_size == 1024**2+async def _check_GET_x10(self,client: H2ClientProtocol,