Scrapy, a fast high-level web crawling & scraping framework for Python.
Add more header-related tests. (#8375)
Andrey Rakhmatullinddf89e92f7ec
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 29e7238bd..c6b19d464 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -233,6 +233,24 @@ class StorageTestMixin(TestBase):assert cached_response.encoding == "utf-8"assert cached_response.text == response.text+ def test_storage_headers(self) -> None:+ """Repeated headers, the case of header names and header values that+ are not valid UTF-8 are kept."""+ with self._storage() as (storage, crawler):+ response = Response(+ "http://www.example.com",+ headers=[+ ("Set-Cookie", "a=1"),+ ("content-type", "text/html"),+ ("set-cookie", "b=2"),+ ("X-Custom-Header", b"caf\xe9"),+ ],+ body=b"test body",+ )+ storage.store_response(crawler.spider, self.request, response)+ cached_response = storage.retrieve_response(crawler.spider, self.request)+ self.assertEqualResponse(response, cached_response)+def test_storage_old_cache_entry(self):with self._storage() as (storage, crawler):storage.store_response(crawler.spider, self.request, self.response)@@ -849,6 +867,27 @@ class FilesystemStorageTestMixin(StorageTestMixin):rpath = Path(storage._get_request_path(spider, request))(rpath / "response_data").unlink()+ @pytest.mark.parametrize("filename", ["request_headers", "response_headers"])+ def test_storage_header_file(self, filename: str) -> None:+ headers = [+ ("X-Repeated", "1"),+ ("content-type", "text/html"),+ ("x-repeated", "2"),+ ("X-Removed", None),+ ("X-Custom-Header", b"caf\xe9"),+ ]+ expected = (+ b"X-Repeated: 1\r\nX-Repeated: 2\r\ncontent-type: text/html\r\n"+ b"X-Custom-Header: caf\xe9"+ )+ request = Request("http://www.example.com", headers=headers)+ response = Response("http://www.example.com", headers=headers)+ with self._storage() as (storage, crawler):+ storage.store_response(crawler.spider, request, response)+ rpath = Path(storage._get_request_path(crawler.spider, request))+ with storage._open(rpath / filename, "rb") as f:+ assert f.read() == expected+class DbmStorageTestMixin(StorageTestMixin):storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 3fc800dc2..60b69da84 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -97,6 +97,17 @@ def test_delete_and_contains() -> None:assert "Content-Type" not in h+def test_pop() -> None:+ h = Headers({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})+ olist = h.getlist("X-Forwarded-For")+ assert h.pop("x-forwarded-for") is olist+ assert olist == [b"ip1", b"ip2"]+ assert list(h.keys()) == [b"Content-Type"]+ assert h.pop("X-Forwarded-For", None) is None+ with pytest.raises(KeyError):+ h.pop("X-Forwarded-For")++def test_setdefault() -> None:h = Headers()hlist = ["ip1", "ip2"]@@ -217,6 +228,26 @@ def test_invalid_value() -> None:Headers()["foo"] = [object()]+ ("headers", "expected"),+ [+ ([], b""),+ ([("X-Removed", None)], b""),+ (+ [+ ("Content-Type", "text/html"),+ ("set-cookie", "a=1"),+ ("X-Removed", None),+ ("Set-Cookie", "b=2"),+ ],+ b"Content-Type: text/html\r\nset-cookie: a=1\r\nset-cookie: b=2",+ ),+ ],+)+def test_to_string(headers: list[tuple[str, str | None]], expected: bytes) -> None:+ assert Headers(headers).to_string() == expected++def test_to_unicode_dict_undecodable() -> None:h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})with pytest.warns(ScrapyDeprecationWarning, match="to_unicode_dict"):diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 9d3ced599..8fc6e16be 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -151,6 +151,16 @@ class TestFingerprint:b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11",{"include_headers": ["A"], "keep_fragments": True},),+ (+ Request("https://example.org", headers=[("A", b"B"), ("a", b"C")]),+ b"\xf6\xd4\xfe\xb0e\xa4\xde\xab\xfc*\x83o=\xa8\x8a\x8f#\x9e^z",+ {"include_headers": ["A"]},+ ),+ (+ Request("https://example.org", headers={"A": None}),+ b"*\x1f\xf2\xf5a\xc2?\xda9\x11>\xff$\xed\xc2\x81\xa1J(\xae",+ {"include_headers": ["A"]},+ ),(Request("https://example.org/ab"),b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6",