Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a HTTP2_MAX_FRAME_SIZE setting (#7988)

Adriana22523bfd3d4
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index d95e7b52b..2d04b621c 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -209,12 +209,8 @@ Known limitations of the HTTP/2 support:
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
-- No setting to specify a maximum `frame size`_ larger than the default
- value, 16384. Connections to servers that send a larger frame will fail.
-
- No support for `server pushes`_, which are ignored.
-.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 010c9d1c6..e58815fad 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1499,6 +1499,28 @@ Default: ``None``
The Project ID that will be used when storing data on `Google Cloud Storage`_.
+.. setting:: HTTP2_MAX_FRAME_SIZE
+
+HTTP2_MAX_FRAME_SIZE
+--------------------
+
+.. versionadded:: VERSION
+
+Default: ``16384``
+
+Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and
+``16777215``. Connections to servers that send a larger frame fail.
+
+Raise it for servers that send larger frames regardless of this value. Note
+that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked
+once per received frame, so a higher value allows a response to exceed them by
+more before being caught.
+
+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores
+this setting, as ``httpx`` does not allow configuring the frame size.
+
+.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
+
.. setting:: ITEM_PIPELINES
ITEM_PIPELINES
diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.py
index aff985d94..1e673ce8b 100644
--- a/scrapy/core/_http2/protocol.py
+++ b/scrapy/core/_http2/protocol.py
@@ -21,6 +21,7 @@ from h2.events import (
WindowUpdated,
)
from h2.exceptions import FrameTooLargeError, H2Error
+from h2.settings import SettingCodes
from twisted.internet.interfaces import (
IAddress,
IHandshakeListener,
@@ -261,6 +262,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# Initiate H2 Connection
self.conn.initiate_connection()
+ max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE")
+ if max_frame_size != self.conn.local_settings.max_frame_size:
+ self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size})
self._write_to_transport()
def _lose_connection_with_error(self, errors: list[BaseException]) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d7b30b849..b97844103 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -108,6 +108,7 @@ __all__ = [
"FTP_PASSWORD",
"FTP_USER",
"GCS_PROJECT_ID",
+ "HTTP2_MAX_FRAME_SIZE",
"HTTPAUTH_DOMAIN",
"HTTPAUTH_PASS",
"HTTPAUTH_USER",
@@ -402,6 +403,8 @@ FTP_PASSWORD = "guest" # noqa: S105
GCS_PROJECT_ID = None
+HTTP2_MAX_FRAME_SIZE = 16384
+
HTTPAUTH_USER = ""
HTTPAUTH_PASS = ""
HTTPAUTH_DOMAIN = None
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 161015231..075f1acb9 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -37,6 +37,7 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
from scrapy.core._http2.protocol import H2ClientProtocol
+ from scrapy.crawler import Crawler
pytestmark = [
@@ -236,9 +237,16 @@ class TestHttps2ClientProtocol:
) + self.certificate_file.read_text(encoding="utf-8")
return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return]
+ @pytest.fixture
+ def crawler(self, request: pytest.FixtureRequest) -> Crawler:
+ return get_crawler(settings_dict=getattr(request, "param", None))
+
@async_yield_fixture # type: ignore[untyped-decorator]
async def client(
- self, server_port: int, client_certificate: PrivateCertificate
+ self,
+ server_port: int,
+ client_certificate: PrivateCertificate,
+ crawler: Crawler,
) -> AsyncGenerator[H2ClientProtocol]:
from twisted.internet import reactor
@@ -250,7 +258,7 @@ class TestHttps2ClientProtocol:
acceptableProtocols=[b"h2"],
)
uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8"))
- h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred())
+ h2_client_factory = H2ClientFactory(uri, crawler, Deferred())
client_endpoint = SSL4ClientEndpoint(
reactor, self.host, server_port, client_options
)
@@ -312,6 +320,17 @@ class TestHttps2ClientProtocol:
request = Request(self.get_url(server_port, "/get-data-html-large"))
await self._check_GET(client, request, Data.HTML_LARGE, 200)
+ @pytest.mark.parametrize(
+ "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True
+ )
+ @deferred_f_from_coro_f
+ async def test_GET_large_frames(
+ self, server_port: int, client: H2ClientProtocol
+ ) -> None:
+ request = Request(self.get_url(server_port, "/get-data-html-large"))
+ await self._check_GET(client, request, Data.HTML_LARGE, 200)
+ assert client.conn.local_settings.max_frame_size == 1024**2
+
async def _check_GET_x10(
self,
client: H2ClientProtocol,