Scrapy, a fast high-level web crawling & scraping framework for Python.
Keep the case of header names (#7893)
Adrianb5cde701c3fb
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex ba8bf1174..aa6bef579 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -140,36 +140,44 @@ individual handler docs for more differences:- Proxies- Bad headers- TLS+ - Header case* - :class:`Aiohttp <scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler>`- asyncio- 1.1- HTTP- Fail- Stdlib ``ssl``+ - As written* - :class:`H2 <scrapy.core.downloader.handlers.http2.H2DownloadHandler>`- Reactor, :ref:`twisted-http2 <extras>` extra- 2- None- Not applicable- ``cryptography``+ - Lowercase* - :class:`HTTP11 <scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler>`- Reactor- 1.1- HTTP- Skip bad- ``cryptography``+ - Capitalized* - :class:`Httpx <scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler>`- asyncio, :ref:`httpx <extras>` extra- 1.1, 2- HTTP, SOCKS- Fail- Stdlib ``ssl``+ - As writtenBad header handling is what a handler does when a response has a bad headerline, e.g. one with no colon in it, which some servers send. Handlers that skipbad header lines, like web browsers do, still parse the header lines that followthem; other handlers also lose those, or cannot download such responses at all.+Because HTTP/2 requires lowercase header names, handlers only keep the case of+your header names over HTTP/1.1.+You can find additional HTTP download handlers in thescrapy-download-handlers-incubator_ package. This package is made by the Scrapydevelopers and contains experimental handlers that may be included in somediff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 39f8701f1..473aedf63 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -107,6 +107,13 @@ Request objectsA dictionary-like (:class:`scrapy.http.headers.Headers`) object which containsthe request headers.+ .. versionchanged:: VERSION+ Header names keep the case you write them in, instead of being+ converted to ``Title-Case``.++ Lookups are case-insensitive. Whether your case reaches the server+ depends on the :ref:`download handler <download-handlers-ref>`.+.. attribute:: Request.bodyThe request body as bytes.diff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.pyindex 4e078b1b9..df895554c 100644--- a/scrapy/core/_http2/stream.py+++ b/scrapy/core/_http2/stream.py@@ -247,11 +247,10 @@ class Stream:content_length = str(len(self._request.body))headers.append(("Content-Length", content_length))- content_length_name = self._request.headers.normkey(b"Content-Length")for name, values in self._request.headers.items():for value_bytes in values:value = str(value_bytes, "utf-8")- if name == content_length_name:+ if name.lower() == b"content-length":if value != content_length:logger.warning("Ignoring bad Content-Length header %r of request %r, "diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 1d138254c..7b1917bf6 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -184,7 +184,7 @@ class HttpxDownloadHandler(_Base):@staticmethoddef _extract_headers(response: httpx.Response) -> Headers:- return Headers(response.headers.multi_items())+ return Headers(response.headers.raw)@staticmethoddef _build_base_response_args(diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 3155302d2..05bd0ee5d 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -126,7 +126,7 @@ class FTPDownloadHandler(BaseDownloadHandler):protocol.close()assert client.transportclient.transport.loseConnection()- headers = {"local filename": protocol.filename or b"", "size": protocol.size}+ headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size}body = protocol.filename or protocol.body.read()respcls = responsetypes.from_args(url=request.url, body=body)return respcls(url=request.url, status=200, body=body, headers=headers)diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex fb26d5f18..27063b377 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -65,13 +65,26 @@ class Headers(dict): # type: ignore[type-arg]) -> None:items = seq.items() if isinstance(seq, Mapping) else seqiseq: dict[bytes, list[bytes]] = {}+ # normkey() only sees keys already stored, so keys from seq that differ+ # only in case are mapped to a single spelling here.+ spellings: dict[bytes, bytes] = {}for k, v in items:- iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))+ key = self.normkey(k)+ key = spellings.setdefault(key.lower(), key)+ iseq.setdefault(key, []).extend(self.normvalue(v))dict.update(self, iseq)def normkey(self, key: str | bytes) -> bytes:- """Normalize key to bytes"""- return self._tobytes(key.title())+ """Normalize key to bytes, matching the case of an existing key if any"""+ key = self._tobytes(key)+ if dict.__contains__(self, key):+ return key+ lower_key = key.lower()+ existing_key: bytes+ for existing_key in dict.keys(self):+ if existing_key.lower() == lower_key:+ return existing_key+ return keydef normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:"""Normalize values to bytes"""diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 10b13607e..9582b4298 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -792,7 +792,9 @@ class TestHttps2ClientProtocol:response_headers = json.loads(str(response.body, "utf-8"))assert isinstance(response_headers, dict)+ # The server reports header names in its own case.+ received = {k.lower(): v for k, v in response_headers.items()}for k, v in request.headers.items():- k_decoded, v_decoded = str(k, "utf-8"), str(v[0], "utf-8")- assert k_decoded in response_headers- assert v_decoded == response_headers[k_decoded]+ k_decoded, v_decoded = str(k, "utf-8").lower(), str(v[0], "utf-8")+ assert k_decoded in received+ assert v_decoded == received[k_decoded]diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 514ab2513..a479edfee 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -67,6 +67,27 @@ def test_encode_multiple() -> None:assert val[0] == b"\xc2\xa3"+def test_key_case_kept() -> None:+ h = Headers({"accept": "text/html", "access_token": "foo"})+ assert sorted(h.keys()) == [b"accept", b"access_token"]+++def test_key_case_of_first_spelling_wins() -> None:+ h = Headers({"accept": "a", "Accept": "b"})+ assert h.getlist("ACCEPT") == [b"a", b"b"]+ assert list(h.keys()) == [b"accept"]++ h["ACCEPT"] = "c"+ h.appendlist("aCCept", "d")+ h.update({"ACCEPT": "e"})+ assert list(h.keys()) == [b"accept"]+ assert h.getlist("accept") == [b"e"]++ del h["ACCEPT"]+ h["ACCEPT"] = "f"+ assert list(h.keys()) == [b"ACCEPT"]++def test_delete_and_contains() -> None:h = Headers()h["Content-Type"] = "text/html"diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 6ca608053..08e03970c 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -39,7 +39,7 @@ if TYPE_CHECKING:headers={"Content-type": b"text/html"},body=b"Some body",),- b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-type: text/html\r\n\r\nSome body",),],)diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex 914b8ce80..aad138513 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -330,12 +330,12 @@ class TestHttpBase(ABC):async with self.get_dh() as download_handler:response = await download_handler.download_request(request)assert response.status == 200- received_headers = set(response.headers.keys())+ received_headers = {key.lower() for key in response.headers}allowed_headers = {- b"Content-Length",- b"Content-Type",- b"Date",- b"Server",+ b"content-length",+ b"content-type",+ b"date",+ b"server",}extra_headers = received_headers - allowed_headersassert not extra_headers, response.headers