Scrapy, a fast high-level web crawling & scraping framework for Python.
Copy header values in Request.to_dict() and Response.to_dict(). (#8269)
Andrey Rakhmatulline204ad090690
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 05ed8955b..7cff5f34d 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -422,7 +422,7 @@ class Request(object_ref):if callable(self.errback)else self.errback),- "headers": dict(self.headers),+ "headers": {k: list(v) for k, v in self.headers.items()},}for attr in self.attributes:d.setdefault(attr, getattr(self, attr))diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex b8bd858f5..4140d104d 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -204,7 +204,7 @@ class Response(object_ref):so subclasses only need to override this method, and :meth:`from_dict`,if some of their attributes cannot be stored as is."""- d: dict[str, Any] = {"headers": dict(self.headers)}+ d: dict[str, Any] = {"headers": {k: list(v) for k, v in self.headers.items()}}for attr in self.attributes:if attr in {"request", "certificate"}:continuediff --git a/tests/test_request_dict.py b/tests/test_request_dict.pyindex f85be8977..a34cc754e 100644--- a/tests/test_request_dict.py+++ b/tests/test_request_dict.py@@ -71,6 +71,19 @@ def test_all_attributes(spider: MethodsSpider) -> None:_assert_serializes_ok(r, spider=spider)+def test_headers() -> None:+ r = Request(+ "http://www.example.com",+ headers=[("X-Multi", "a"), ("x-multi", "b"), ("dnt", "1"), ("Accept", None)],+ )+ headers = r.to_dict()["headers"]+ assert type(headers) is dict+ assert headers == {b"X-Multi": [b"a", b"b"], b"dnt": [b"1"], b"Accept": []}+ headers[b"dnt"].append(b"2")+ assert r.headers.getlist("dnt") == [b"1"]+ _assert_serializes_ok(r)++def test_latin1_body() -> None:r = Request("http://www.example.com", body=b"\xa3")_assert_serializes_ok(r)diff --git a/tests/test_response_dict.py b/tests/test_response_dict.pyindex e258a2316..0ec87d82e 100644--- a/tests/test_response_dict.py+++ b/tests/test_response_dict.py@@ -55,6 +55,20 @@ def test_all_attributes() -> None:assert getattr(response2, attribute) == getattr(response, attribute)+def test_headers() -> None:+ response = Response(+ "https://example.com",+ headers=[("Set-Cookie", "a=1"), ("set-cookie", "b=2"), ("etag", "x")],+ )+ headers = response.to_dict()["headers"]+ assert type(headers) is dict+ assert headers == {b"Set-Cookie": [b"a=1", b"b=2"], b"etag": [b"x"]}+ headers[b"etag"].append(b"y")+ assert response.headers.getlist("etag") == [b"x"]+ response2 = response_from_dict(response.to_dict())+ assert response2.headers == response.headers++def test_custom_attributes() -> None:response = CustomResponse("https://example.com", custom="value")response2 = response_from_dict(response.to_dict())diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex d944ef097..46166815a 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -62,6 +62,16 @@ class TestRequestQueueBase(ABC):assert q.peek() is Noneassert q.pop() is None+ def test_headers(self, q: queuelib.queue.BaseQueue):+ req = Request(+ "http://www.example.com",+ headers=[("X-Multi", "a"), ("x-multi", "b"), ("Accept", None)],+ )+ q.push(req)+ result = q.pop()+ assert result is not None+ assert result.headers == req.headers+def test_order(self, q: queuelib.queue.BaseQueue):assert len(q) == 0assert q.peek() is None