Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Keep the case of header names (#7893)

Adrianb5cde701c3fb
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index ba8bf1174..aa6bef579 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -140,36 +140,44 @@ individual handler docs for more differences:
- Proxies
- Bad headers
- TLS
+ - Header case
* - :class:`Aiohttp <scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler>`
- asyncio
- 1.1
- HTTP
- Fail
- Stdlib ``ssl``
+ - As written
* - :class:`H2 <scrapy.core.downloader.handlers.http2.H2DownloadHandler>`
- Reactor, :ref:`twisted-http2 <extras>` extra
- 2
- None
- Not applicable
- ``cryptography``
+ - Lowercase
* - :class:`HTTP11 <scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler>`
- Reactor
- 1.1
- HTTP
- Skip bad
- ``cryptography``
+ - Capitalized
* - :class:`Httpx <scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler>`
- asyncio, :ref:`httpx <extras>` extra
- 1.1, 2
- HTTP, SOCKS
- Fail
- Stdlib ``ssl``
+ - As written
Bad header handling is what a handler does when a response has a bad header
line, e.g. one with no colon in it, which some servers send. Handlers that skip
bad header lines, like web browsers do, still parse the header lines that follow
them; other handlers also lose those, or cannot download such responses at all.
+Because HTTP/2 requires lowercase header names, handlers only keep the case of
+your header names over HTTP/1.1.
+
You can find additional HTTP download handlers in the
scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
developers and contains experimental handlers that may be included in some
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 39f8701f1..473aedf63 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -107,6 +107,13 @@ Request objects
A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains
the request headers.
+ .. versionchanged:: VERSION
+ Header names keep the case you write them in, instead of being
+ converted to ``Title-Case``.
+
+ Lookups are case-insensitive. Whether your case reaches the server
+ depends on the :ref:`download handler <download-handlers-ref>`.
+
.. attribute:: Request.body
The request body as bytes.
diff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.py
index 4e078b1b9..df895554c 100644
--- a/scrapy/core/_http2/stream.py
+++ b/scrapy/core/_http2/stream.py
@@ -247,11 +247,10 @@ class Stream:
content_length = str(len(self._request.body))
headers.append(("Content-Length", content_length))
- content_length_name = self._request.headers.normkey(b"Content-Length")
for name, values in self._request.headers.items():
for value_bytes in values:
value = str(value_bytes, "utf-8")
- if name == content_length_name:
+ if name.lower() == b"content-length":
if value != content_length:
logger.warning(
"Ignoring bad Content-Length header %r of request %r, "
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 1d138254c..7b1917bf6 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -184,7 +184,7 @@ class HttpxDownloadHandler(_Base):
@staticmethod
def _extract_headers(response: httpx.Response) -> Headers:
- return Headers(response.headers.multi_items())
+ return Headers(response.headers.raw)
@staticmethod
def _build_base_response_args(
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 3155302d2..05bd0ee5d 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -126,7 +126,7 @@ class FTPDownloadHandler(BaseDownloadHandler):
protocol.close()
assert client.transport
client.transport.loseConnection()
- headers = {"local filename": protocol.filename or b"", "size": protocol.size}
+ headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index fb26d5f18..27063b377 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -65,13 +65,26 @@ class Headers(dict): # type: ignore[type-arg]
) -> None:
items = seq.items() if isinstance(seq, Mapping) else seq
iseq: dict[bytes, list[bytes]] = {}
+ # normkey() only sees keys already stored, so keys from seq that differ
+ # only in case are mapped to a single spelling here.
+ spellings: dict[bytes, bytes] = {}
for k, v in items:
- iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+ key = self.normkey(k)
+ key = spellings.setdefault(key.lower(), key)
+ iseq.setdefault(key, []).extend(self.normvalue(v))
dict.update(self, iseq)
def normkey(self, key: str | bytes) -> bytes:
- """Normalize key to bytes"""
- return self._tobytes(key.title())
+ """Normalize key to bytes, matching the case of an existing key if any"""
+ key = self._tobytes(key)
+ if dict.__contains__(self, key):
+ return key
+ lower_key = key.lower()
+ existing_key: bytes
+ for existing_key in dict.keys(self):
+ if existing_key.lower() == lower_key:
+ return existing_key
+ return key
def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:
"""Normalize values to bytes"""
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 10b13607e..9582b4298 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -792,7 +792,9 @@ class TestHttps2ClientProtocol:
response_headers = json.loads(str(response.body, "utf-8"))
assert isinstance(response_headers, dict)
+ # The server reports header names in its own case.
+ received = {k.lower(): v for k, v in response_headers.items()}
for k, v in request.headers.items():
- k_decoded, v_decoded = str(k, "utf-8"), str(v[0], "utf-8")
- assert k_decoded in response_headers
- assert v_decoded == response_headers[k_decoded]
+ k_decoded, v_decoded = str(k, "utf-8").lower(), str(v[0], "utf-8")
+ assert k_decoded in received
+ assert v_decoded == received[k_decoded]
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 514ab2513..a479edfee 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -67,6 +67,27 @@ def test_encode_multiple() -> None:
assert val[0] == b"\xc2\xa3"
+def test_key_case_kept() -> None:
+ h = Headers({"accept": "text/html", "access_token": "foo"})
+ assert sorted(h.keys()) == [b"accept", b"access_token"]
+
+
+def test_key_case_of_first_spelling_wins() -> None:
+ h = Headers({"accept": "a", "Accept": "b"})
+ assert h.getlist("ACCEPT") == [b"a", b"b"]
+ assert list(h.keys()) == [b"accept"]
+
+ h["ACCEPT"] = "c"
+ h.appendlist("aCCept", "d")
+ h.update({"ACCEPT": "e"})
+ assert list(h.keys()) == [b"accept"]
+ assert h.getlist("accept") == [b"e"]
+
+ del h["ACCEPT"]
+ h["ACCEPT"] = "f"
+ assert list(h.keys()) == [b"ACCEPT"]
+
+
def test_delete_and_contains() -> None:
h = Headers()
h["Content-Type"] = "text/html"
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 6ca608053..08e03970c 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -39,7 +39,7 @@ if TYPE_CHECKING:
headers={"Content-type": b"text/html"},
body=b"Some body",
),
- b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-type: text/html\r\n\r\nSome body",
),
],
)
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index 914b8ce80..aad138513 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -330,12 +330,12 @@ class TestHttpBase(ABC):
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.status == 200
- received_headers = set(response.headers.keys())
+ received_headers = {key.lower() for key in response.headers}
allowed_headers = {
- b"Content-Length",
- b"Content-Type",
- b"Date",
- b"Server",
+ b"content-length",
+ b"content-type",
+ b"date",
+ b"server",
}
extra_headers = received_headers - allowed_headers
assert not extra_headers, response.headers