Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Honor user-set Content-Length request headers on HTTP/2 (#7944)

Adrianf1a56c046fb7
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index b7cd9f330..4209385f0 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -333,6 +333,9 @@ Other limitations:
- HTTPS proxies to HTTPS destinations are not supported.
+- A ``Content-Length`` request header is sent in addition to the one built
+ from the request body, and most servers reject requests with two.
+
.. _httpx-handler:
HttpxDownloadHandler
diff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.py
index df895554c..c6b004472 100644
--- a/scrapy/core/_http2/stream.py
+++ b/scrapy/core/_http2/stream.py
@@ -244,23 +244,12 @@ class Stream:
(":path", path),
]
- content_length = str(len(self._request.body))
- headers.append(("Content-Length", content_length))
+ if b"Content-Length" not in self._request.headers:
+ headers.append(("Content-Length", str(len(self._request.body))))
for name, values in self._request.headers.items():
for value_bytes in values:
- value = str(value_bytes, "utf-8")
- if name.lower() == b"content-length":
- if value != content_length:
- logger.warning(
- "Ignoring bad Content-Length header %r of request %r, "
- "sending %r instead",
- value,
- self._request,
- content_length,
- )
- continue
- headers.append((str(name, "utf-8"), value))
+ headers.append((str(name, "utf-8"), str(value_bytes, "utf-8")))
return headers
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index c7e128a22..f81d617ae 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -61,6 +61,8 @@ class HTTP11DownloadHandlerMixin:
}
}
+ handler_supports_custom_content_length = False
+
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 1d2d18d0f..37da87ee8 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-import logging
import sys
from typing import TYPE_CHECKING, Any
@@ -126,35 +125,12 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
assert response.body == b""
@coroutine_test
- async def test_custom_content_length_good(self, mockserver: MockServer) -> None:
+ async def test_custom_content_length_bad(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
- custom_content_length = str(len(request.body))
- request.headers["Content-Length"] = custom_content_length
+ request.headers["Content-Length"] = str(len(request.body) + 1)
async with self.get_dh() as download_handler:
- response = await download_handler.download_request(request)
- assert response.text == custom_content_length
-
- @coroutine_test
- async def test_custom_content_length_bad(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
- ) -> None:
- request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
- actual_content_length = str(len(request.body))
- bad_content_length = str(len(request.body) + 1)
- request.headers["Content-Length"] = bad_content_length
- async with self.get_dh() as download_handler:
- with caplog.at_level(logging.DEBUG):
- response = await download_handler.download_request(request)
- assert response.text == actual_content_length
- assert (
- "scrapy.core._http2.stream",
- logging.WARNING,
- (
- f"Ignoring bad Content-Length header "
- f"{bad_content_length!r} of request {request}, sending "
- f"{actual_content_length!r} instead"
- ),
- ) in caplog.record_tuples
+ with pytest.raises(DownloadFailedError):
+ await download_handler.download_request(request)
@coroutine_test
async def test_data_loss_handling(self, mockserver: MockServer) -> None:
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index dd3aff629..dd807fd95 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -75,6 +75,9 @@ class TestHttpBase(ABC):
# h2.connection.H2Connection.receive_data()), thus closing all streams that
# were using it, and we handle this as a normal exception.
handler_supports_http2_dataloss: bool = True
+ # whether the handler sends a request Content-Length header as is, instead
+ # of building its own (https://github.com/scrapy/scrapy/issues/4919)
+ handler_supports_custom_content_length: bool = True
# whether the handler can request hostnames that the idna package rejects
# (see IDNA_REJECTED_HOSTNAMES)
handler_supports_idna_rejected_hostnames: bool = True
@@ -478,6 +481,40 @@ class TestHttpBase(ABC):
assert len(contentlengths) == 1
assert contentlengths == [b"0"]
+ @coroutine_test
+ async def test_custom_content_length(self, mockserver: MockServer) -> None:
+ body = b"1" * 100
+ request = Request(
+ mockserver.url("/echo", is_secure=self.is_secure),
+ method="POST",
+ body=body,
+ headers={"Content-Length": str(len(body))},
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ if not self.handler_supports_custom_content_length:
+ # The server gets two Content-Length headers and rejects the
+ # request.
+ assert response.status == 400
+ return
+ echo = json.loads(response.text)
+ assert Headers(echo["headers"]).getlist("Content-Length") == [b"100"]
+ assert echo["body"] == body.decode()
+
+ @coroutine_test
+ async def test_custom_content_length_bodyless(self, mockserver: MockServer) -> None:
+ request = Request(
+ mockserver.url("/contentlength", is_secure=self.is_secure),
+ method="POST",
+ headers={"Content-Length": "0"},
+ )
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ if not self.handler_supports_custom_content_length:
+ assert response.status == 400
+ return
+ assert response.body == b"0"
+
@coroutine_test
async def test_payload(self, mockserver: MockServer) -> None:
body = b"1" * 100 # PayloadResource requires body length to be 100