Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate more Headers methods. (#8264)

Andrey Rakhmatullinb532f2e34ecd
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 42c23abfb..68ec4a931 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
@@ -10,8 +10,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
- from collections.abc import Mapping
-
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.http import Response
@@ -68,10 +66,15 @@ class S3DownloadHandler(BaseDownloadHandler):
else:
import botocore.awsrequest # noqa: PLC0415
+ # AWSRequest takes a mapping, so values of the same header are
+ # joined with commas.
+ headers: dict[str, list[str]] = {}
+ for name, value in request.headers.to_tuple_list():
+ headers.setdefault(name, []).append(value)
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
- headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),
+ headers={name: ",".join(values) for name, values in headers.items()},
data=request.body,
)
assert self._signer
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 7bb126cb1..b880454ee 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -42,7 +42,13 @@ class Headers(dict): # type: ignore[type-arg]
def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
return dict.__contains__(self, self._normkey(key))
- has_key = __contains__
+ def has_key(self, key: str | bytes) -> bool:
+ warnings.warn(
+ 'Headers.has_key() is deprecated, use the "in" operator instead.',
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return key in self
def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.setdefault(self, self._normkey(key), self._normvalue(def_val))
@@ -51,6 +57,12 @@ class Headers(dict): # type: ignore[type-arg]
def fromkeys( # type: ignore[override]
cls, keys: Iterable[str | bytes], value: Any = None
) -> Self:
+ warnings.warn(
+ "Headers.fromkeys() is deprecated, pass (key, value) pairs to"
+ " Headers() instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return cls((k, value) for k in keys)
def pop(self, key: str | bytes, *args: Any) -> Any:
@@ -152,11 +164,22 @@ class Headers(dict): # type: ignore[type-arg]
return []
def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
+ warnings.warn(
+ "Headers.setlist() is deprecated, assign the list instead:"
+ " headers[key] = values.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self[key] = list_
def setlistdefault(
self, key: str | bytes, default_list: Iterable[_RawValue] = ()
) -> Any:
+ warnings.warn(
+ "Headers.setlistdefault() is deprecated, use setdefault() instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return self.setdefault(key, default_list)
def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
@@ -177,10 +200,11 @@ class Headers(dict): # type: ignore[type-arg]
return headers_dict_to_raw(self)
def to_unicode_dict(self) -> CaseInsensitiveDict:
- """Return headers as a CaseInsensitiveDict with str keys
- and str values. Multiple values are joined with ','. Bytes that
- cannot be decoded are replaced with U+FFFD.
- """
+ warnings.warn(
+ "Headers.to_unicode_dict() is deprecated, use to_tuple_list() instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return CaseInsensitiveDict(
(
to_unicode(key, encoding=self.encoding, errors="replace"),
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index a479edfee..7e4932601 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -2,6 +2,7 @@ import copy
import pytest
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
@@ -109,8 +110,15 @@ def test_setdefault() -> None:
assert h.getlist("X-Forwarded-For") is olist
+def test_has_key() -> None:
+ h = Headers({"Content-Type": "text/html"})
+ with pytest.warns(ScrapyDeprecationWarning, match="has_key"):
+ assert h.has_key("content-type")
+
+
def test_fromkeys() -> None:
- h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")
+ with pytest.warns(ScrapyDeprecationWarning, match="fromkeys"):
+ h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")
assert h.getlist("Content-Type") == [b"value"]
assert h.getlist("Content-Length") == [b"value"]
@@ -161,14 +169,17 @@ def test_appendlist() -> None:
def test_setlist() -> None:
h1 = Headers({"header1": "value1"})
assert h1.getlist("header1") == [b"value1"]
- h1.setlist("header1", [b"value2", b"value3"])
+ with pytest.warns(ScrapyDeprecationWarning, match="setlist"):
+ h1.setlist("header1", [b"value2", b"value3"])
assert h1.getlist("header1") == [b"value2", b"value3"]
def test_setlistdefault() -> None:
h1 = Headers({"header1": "value1"})
- h1.setlistdefault("header1", ["value2", "value3"])
- h1.setlistdefault("header2", ["value2", "value3"])
+ with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):
+ h1.setlistdefault("header1", ["value2", "value3"])
+ with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):
+ h1.setlistdefault("header2", ["value2", "value3"])
assert h1.getlist("header1") == [b"value1"]
assert h1.getlist("header2") == [b"value2", b"value3"]
@@ -187,7 +198,7 @@ def test_int_value() -> None:
h1 = Headers({"hey": 5})
h1["foo"] = 1
h1.setdefault("bar", 2)
- h1.setlist("buz", [1, "dos", 3])
+ h1["buz"] = [1, "dos", 3]
assert h1.getlist("foo") == [b"1"]
assert h1.getlist("bar") == [b"2"]
assert h1.getlist("buz") == [b"1", b"dos", b"3"]
@@ -202,12 +213,14 @@ def test_invalid_value() -> None:
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setdefault("foo", object())
with pytest.raises(TypeError, match="Unsupported value type"):
- Headers().setlist("foo", [object()]) # type: ignore[list-item]
+ Headers()["foo"] = [object()]
def test_to_unicode_dict_undecodable() -> None:
h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})
- assert h.to_unicode_dict() == {
+ with pytest.warns(ScrapyDeprecationWarning, match="to_unicode_dict"):
+ unicode_dict = h.to_unicode_dict()
+ assert unicode_dict == {
"Public-Key-Pins": 'pin-sha256=�"a"',
"X-�": "ok",
}
diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py
index fea7f9f45..209bc30f3 100644
--- a/tests/utils/bases/redirect.py
+++ b/tests/utils/bases/redirect.py
@@ -203,7 +203,7 @@ class TestRedirectBase(ABC):
assert {
**safe_headers,
**cookie_header,
- } == different_port_redirect_request.headers.to_unicode_dict()
+ } == dict(different_port_redirect_request.headers.to_tuple_list())
# A domain change drops both the Authorization and the Cookie header.
external_response = self.get_response(original_request, "https://example.org/a")
@@ -211,7 +211,7 @@ class TestRedirectBase(ABC):
original_request, external_response
)
assert isinstance(external_redirect_request, Request)
- assert safe_headers == external_redirect_request.headers.to_unicode_dict()
+ assert safe_headers == dict(external_redirect_request.headers.to_tuple_list())
# A scheme upgrade (http → https) drops the Authorization header
# because the origin changes, but keeps the Cookie header because the
@@ -224,7 +224,7 @@ class TestRedirectBase(ABC):
assert {
**safe_headers,
**cookie_header,
- } == upgrade_redirect_request.headers.to_unicode_dict()
+ } == dict(upgrade_redirect_request.headers.to_tuple_list())
# A scheme downgrade (https → http) drops the Authorization header
# because the origin changes, and the Cookie header because its value
@@ -240,7 +240,7 @@ class TestRedirectBase(ABC):
original_request, downgrade_response
)
assert isinstance(downgrade_redirect_request, Request)
- assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
+ assert safe_headers == dict(downgrade_redirect_request.headers.to_tuple_list())
def _check_proxy_scenario(
self,