Scrapy, a fast high-level web crawling & scraping framework for Python.
import copy
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
def test_basics() -> None:
h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
assert h["Content-Type"]
assert h["Content-Length"]
with pytest.raises(KeyError):
h["Accept"]
assert h.get("Accept") is None
assert h.getlist("Accept") == []
assert h.get("Accept", "*/*") == b"*/*"
assert h.getlist("Accept", "*/*") == [b"*/*"]
assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [
b"text/html",
b"images/jpeg",
]
def test_single_value() -> None:
h = Headers()
h["Content-Type"] = "text/html"
assert h["Content-Type"] == b"text/html"
assert h.get("Content-Type") == b"text/html"
assert h.getlist("Content-Type") == [b"text/html"]
def test_multivalue() -> None:
h = Headers()
h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]
assert h["X-Forwarded-For"] == b"ip2"
assert h.get("X-Forwarded-For") == b"ip2"
assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
def test_multivalue_for_one_header() -> None:
h = Headers((("a", "b"), ("a", "c")))
assert h["a"] == b"c"
assert h.get("a") == b"c"
assert h.getlist("a") == [b"b", b"c"]
def test_encode_utf8() -> None:
h = Headers({"key": "\xa3"}, encoding="utf-8")
key, val = dict(h.items()).popitem()
assert isinstance(key, bytes), key
assert isinstance(val[0], bytes), val[0]
assert val[0] == b"\xc2\xa3"
def test_encode_latin1() -> None:
h = Headers({"key": "\xa3"}, encoding="latin1")
_, val = dict(h.items()).popitem()
assert val[0] == b"\xa3"
def test_encode_multiple() -> None:
h = Headers({"key": ["\xa3"]}, encoding="utf-8")
_, val = dict(h.items()).popitem()
assert val[0] == b"\xc2\xa3"
def test_key_case_kept() -> None:
h = Headers({"accept": "text/html", "access_token": "foo"})
assert sorted(h.keys()) == [b"accept", b"access_token"]
def test_key_case_of_first_spelling_wins() -> None:
h = Headers({"accept": "a", "Accept": "b"})
assert h.getlist("ACCEPT") == [b"a", b"b"]
assert list(h.keys()) == [b"accept"]
h["ACCEPT"] = "c"
h.appendlist("aCCept", "d")
h.update({"ACCEPT": "e"})
assert list(h.keys()) == [b"accept"]
assert h.getlist("accept") == [b"e"]
del h["ACCEPT"]
h["ACCEPT"] = "f"
assert list(h.keys()) == [b"ACCEPT"]
def test_delete_and_contains() -> None:
h = Headers()
h["Content-Type"] = "text/html"
assert "Content-Type" in h
del h["Content-Type"]
assert "Content-Type" not in h
def test_pop() -> None:
h = Headers({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
olist = h.getlist("X-Forwarded-For")
assert h.pop("x-forwarded-for") is olist
assert olist == [b"ip1", b"ip2"]
assert list(h.keys()) == [b"Content-Type"]
assert h.pop("X-Forwarded-For", None) is None
with pytest.raises(KeyError):
h.pop("X-Forwarded-For")
def test_setdefault() -> None:
h = Headers()
hlist = ["ip1", "ip2"]
olist = h.setdefault("X-Forwarded-For", hlist)
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
assert h.getlist("X-Forwarded-For") is olist
h = Headers()
olist = h.setdefault("X-Forwarded-For", "ip1")
assert h.getlist("X-Forwarded-For") == [b"ip1"]
assert h.getlist("X-Forwarded-For") is olist
def test_has_key() -> None:
h = Headers({"Content-Type": "text/html"})
with pytest.warns(ScrapyDeprecationWarning, match="has_key"):
assert h.has_key("content-type")
def test_fromkeys() -> None:
with pytest.warns(ScrapyDeprecationWarning, match="fromkeys"):
h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")
assert h.getlist("Content-Type") == [b"value"]
assert h.getlist("Content-Length") == [b"value"]
def test_iterables() -> None:
idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
h = Headers(idict)
assert dict(h.items()) == {
b"Content-Type": [b"text/html"],
b"X-Forwarded-For": [b"ip1", b"ip2"],
}
assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
assert sorted(h.items()) == [
(b"Content-Type", [b"text/html"]),
(b"X-Forwarded-For", [b"ip1", b"ip2"]),
]
assert set(h.values()) == {b"ip2", b"text/html"}
def test_update() -> None:
h = Headers()
h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
assert h.getlist("Content-Type") == [b"text/html"]
assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
def test_copy() -> None:
h1 = Headers({"header1": ["value1", "value2"]})
h2 = copy.copy(h1)
assert h1 == h2
assert h1.getlist("header1") == h2.getlist("header1")
assert h1.getlist("header1") is not h2.getlist("header1")
assert isinstance(h2, Headers)
def test_appendlist() -> None:
h1 = Headers({"header1": "value1"})
h1.appendlist("header1", "value3")
assert h1.getlist("header1") == [b"value1", b"value3"]
h1 = Headers()
h1.appendlist("header1", "value1")
h1.appendlist("header1", "value3")
assert h1.getlist("header1") == [b"value1", b"value3"]
def test_setlist() -> None:
h1 = Headers({"header1": "value1"})
assert h1.getlist("header1") == [b"value1"]
with pytest.warns(ScrapyDeprecationWarning, match="setlist"):
h1.setlist("header1", [b"value2", b"value3"])
assert h1.getlist("header1") == [b"value2", b"value3"]
def test_setlistdefault() -> None:
h1 = Headers({"header1": "value1"})
with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):
h1.setlistdefault("header1", ["value2", "value3"])
with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):
h1.setlistdefault("header2", ["value2", "value3"])
assert h1.getlist("header1") == [b"value1"]
assert h1.getlist("header2") == [b"value2", b"value3"]
def test_none_value() -> None:
h1 = Headers()
h1["foo"] = "bar"
h1["foo"] = None
h1.setdefault("foo", "bar")
assert h1["foo"] is None
assert h1.get("foo") is None
assert h1.get("foo", "baz") == b"baz"
assert h1.getlist("foo") == []
def test_int_value() -> None:
h1 = Headers({"hey": 5})
h1["foo"] = 1
h1.setdefault("bar", 2)
h1["buz"] = [1, "dos", 3]
assert h1.getlist("foo") == [b"1"]
assert h1.getlist("bar") == [b"2"]
assert h1.getlist("buz") == [b"1", b"dos", b"3"]
assert h1.getlist("hey") == [b"5"]
def test_invalid_value() -> None:
with pytest.raises(TypeError, match="Unsupported value type"):
Headers({"foo": object()})
with pytest.raises(TypeError, match="Unsupported value type"):
Headers()["foo"] = object()
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setdefault("foo", object())
with pytest.raises(TypeError, match="Unsupported value type"):
Headers()["foo"] = [object()]
@pytest.mark.parametrize(
("headers", "expected"),
[
([], b""),
([("X-Removed", None)], b""),
(
[
("Content-Type", "text/html"),
("set-cookie", "a=1"),
("X-Removed", None),
("Set-Cookie", "b=2"),
],
b"Content-Type: text/html\r\nset-cookie: a=1\r\nset-cookie: b=2",
),
],
)
def test_to_string(headers: list[tuple[str, str | None]], expected: bytes) -> None:
assert Headers(headers).to_string() == expected
def test_to_unicode_dict_undecodable() -> None:
h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})
with pytest.warns(ScrapyDeprecationWarning, match="to_unicode_dict"):
unicode_dict = h.to_unicode_dict()
assert unicode_dict == {
"Public-Key-Pins": 'pin-sha256=�"a"',
"X-�": "ok",
}
def test_to_tuple_list_undecodable() -> None:
h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})
assert h.to_tuple_list() == [
("Public-Key-Pins", 'pin-sha256=�"a"'),
("X-�", "ok"),
]