Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Copy header values in Request.to_dict() and Response.to_dict(). (#8269)

Andrey Rakhmatulline204ad090690
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 05ed8955b..7cff5f34d 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -422,7 +422,7 @@ class Request(object_ref):
if callable(self.errback)
else self.errback
),
- "headers": dict(self.headers),
+ "headers": {k: list(v) for k, v in self.headers.items()},
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b8bd858f5..4140d104d 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -204,7 +204,7 @@ class Response(object_ref):
so subclasses only need to override this method, and :meth:`from_dict`,
if some of their attributes cannot be stored as is.
"""
- d: dict[str, Any] = {"headers": dict(self.headers)}
+ d: dict[str, Any] = {"headers": {k: list(v) for k, v in self.headers.items()}}
for attr in self.attributes:
if attr in {"request", "certificate"}:
continue
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index f85be8977..a34cc754e 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -71,6 +71,19 @@ def test_all_attributes(spider: MethodsSpider) -> None:
_assert_serializes_ok(r, spider=spider)
+def test_headers() -> None:
+ r = Request(
+ "http://www.example.com",
+ headers=[("X-Multi", "a"), ("x-multi", "b"), ("dnt", "1"), ("Accept", None)],
+ )
+ headers = r.to_dict()["headers"]
+ assert type(headers) is dict
+ assert headers == {b"X-Multi": [b"a", b"b"], b"dnt": [b"1"], b"Accept": []}
+ headers[b"dnt"].append(b"2")
+ assert r.headers.getlist("dnt") == [b"1"]
+ _assert_serializes_ok(r)
+
+
def test_latin1_body() -> None:
r = Request("http://www.example.com", body=b"\xa3")
_assert_serializes_ok(r)
diff --git a/tests/test_response_dict.py b/tests/test_response_dict.py
index e258a2316..0ec87d82e 100644
--- a/tests/test_response_dict.py
+++ b/tests/test_response_dict.py
@@ -55,6 +55,20 @@ def test_all_attributes() -> None:
assert getattr(response2, attribute) == getattr(response, attribute)
+def test_headers() -> None:
+ response = Response(
+ "https://example.com",
+ headers=[("Set-Cookie", "a=1"), ("set-cookie", "b=2"), ("etag", "x")],
+ )
+ headers = response.to_dict()["headers"]
+ assert type(headers) is dict
+ assert headers == {b"Set-Cookie": [b"a=1", b"b=2"], b"etag": [b"x"]}
+ headers[b"etag"].append(b"y")
+ assert response.headers.getlist("etag") == [b"x"]
+ response2 = response_from_dict(response.to_dict())
+ assert response2.headers == response.headers
+
+
def test_custom_attributes() -> None:
response = CustomResponse("https://example.com", custom="value")
response2 = response_from_dict(response.to_dict())
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index d944ef097..46166815a 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -62,6 +62,16 @@ class TestRequestQueueBase(ABC):
assert q.peek() is None
assert q.pop() is None
+ def test_headers(self, q: queuelib.queue.BaseQueue):
+ req = Request(
+ "http://www.example.com",
+ headers=[("X-Multi", "a"), ("x-multi", "b"), ("Accept", None)],
+ )
+ q.push(req)
+ result = q.pop()
+ assert result is not None
+ assert result.headers == req.headers
+
def test_order(self, q: queuelib.queue.BaseQueue):
assert len(q) == 0
assert q.peek() is None