Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add more header-related tests. (#8375)

Andrey Rakhmatullinddf89e92f7ec
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 29e7238bd..c6b19d464 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -233,6 +233,24 @@ class StorageTestMixin(TestBase):
assert cached_response.encoding == "utf-8"
assert cached_response.text == response.text
+ def test_storage_headers(self) -> None:
+ """Repeated headers, the case of header names and header values that
+ are not valid UTF-8 are kept."""
+ with self._storage() as (storage, crawler):
+ response = Response(
+ "http://www.example.com",
+ headers=[
+ ("Set-Cookie", "a=1"),
+ ("content-type", "text/html"),
+ ("set-cookie", "b=2"),
+ ("X-Custom-Header", b"caf\xe9"),
+ ],
+ body=b"test body",
+ )
+ storage.store_response(crawler.spider, self.request, response)
+ cached_response = storage.retrieve_response(crawler.spider, self.request)
+ self.assertEqualResponse(response, cached_response)
+
def test_storage_old_cache_entry(self):
with self._storage() as (storage, crawler):
storage.store_response(crawler.spider, self.request, self.response)
@@ -849,6 +867,27 @@ class FilesystemStorageTestMixin(StorageTestMixin):
rpath = Path(storage._get_request_path(spider, request))
(rpath / "response_data").unlink()
+ @pytest.mark.parametrize("filename", ["request_headers", "response_headers"])
+ def test_storage_header_file(self, filename: str) -> None:
+ headers = [
+ ("X-Repeated", "1"),
+ ("content-type", "text/html"),
+ ("x-repeated", "2"),
+ ("X-Removed", None),
+ ("X-Custom-Header", b"caf\xe9"),
+ ]
+ expected = (
+ b"X-Repeated: 1\r\nX-Repeated: 2\r\ncontent-type: text/html\r\n"
+ b"X-Custom-Header: caf\xe9"
+ )
+ request = Request("http://www.example.com", headers=headers)
+ response = Response("http://www.example.com", headers=headers)
+ with self._storage() as (storage, crawler):
+ storage.store_response(crawler.spider, request, response)
+ rpath = Path(storage._get_request_path(crawler.spider, request))
+ with storage._open(rpath / filename, "rb") as f:
+ assert f.read() == expected
+
class DbmStorageTestMixin(StorageTestMixin):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 3fc800dc2..60b69da84 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -97,6 +97,17 @@ def test_delete_and_contains() -> None:
assert "Content-Type" not in h
+def test_pop() -> None:
+ h = Headers({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
+ olist = h.getlist("X-Forwarded-For")
+ assert h.pop("x-forwarded-for") is olist
+ assert olist == [b"ip1", b"ip2"]
+ assert list(h.keys()) == [b"Content-Type"]
+ assert h.pop("X-Forwarded-For", None) is None
+ with pytest.raises(KeyError):
+ h.pop("X-Forwarded-For")
+
+
def test_setdefault() -> None:
h = Headers()
hlist = ["ip1", "ip2"]
@@ -217,6 +228,26 @@ def test_invalid_value() -> None:
Headers()["foo"] = [object()]
+ ("headers", "expected"),
+ [
+ ([], b""),
+ ([("X-Removed", None)], b""),
+ (
+ [
+ ("Content-Type", "text/html"),
+ ("set-cookie", "a=1"),
+ ("X-Removed", None),
+ ("Set-Cookie", "b=2"),
+ ],
+ b"Content-Type: text/html\r\nset-cookie: a=1\r\nset-cookie: b=2",
+ ),
+ ],
+)
+def test_to_string(headers: list[tuple[str, str | None]], expected: bytes) -> None:
+ assert Headers(headers).to_string() == expected
+
+
def test_to_unicode_dict_undecodable() -> None:
h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})
with pytest.warns(ScrapyDeprecationWarning, match="to_unicode_dict"):
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 9d3ced599..8fc6e16be 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -151,6 +151,16 @@ class TestFingerprint:
b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11",
{"include_headers": ["A"], "keep_fragments": True},
),
+ (
+ Request("https://example.org", headers=[("A", b"B"), ("a", b"C")]),
+ b"\xf6\xd4\xfe\xb0e\xa4\xde\xab\xfc*\x83o=\xa8\x8a\x8f#\x9e^z",
+ {"include_headers": ["A"]},
+ ),
+ (
+ Request("https://example.org", headers={"A": None}),
+ b"*\x1f\xf2\xf5a\xc2?\xda9\x11>\xff$\xed\xc2\x81\xa1J(\xae",
+ {"include_headers": ["A"]},
+ ),
(
Request("https://example.org/ab"),
b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6",