Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate more Headers methods. (#8264)
Andrey Rakhmatullinb532f2e34ecd
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 42c23abfb..68ec4a931 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,7 +1,7 @@from __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any, cast+from typing import TYPE_CHECKINGfrom scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning@@ -10,8 +10,6 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:- from collections.abc import Mapping-from scrapy import Requestfrom scrapy.crawler import Crawlerfrom scrapy.http import Response@@ -68,10 +66,15 @@ class S3DownloadHandler(BaseDownloadHandler):else:import botocore.awsrequest # noqa: PLC0415+ # AWSRequest takes a mapping, so values of the same header are+ # joined with commas.+ headers: dict[str, list[str]] = {}+ for name, value in request.headers.to_tuple_list():+ headers.setdefault(name, []).append(value)awsrequest = botocore.awsrequest.AWSRequest(method=request.method,url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",- headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),+ headers={name: ",".join(values) for name, values in headers.items()},data=request.body,)assert self._signerdiff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 7bb126cb1..b880454ee 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -42,7 +42,13 @@ class Headers(dict): # type: ignore[type-arg]def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]return dict.__contains__(self, self._normkey(key))- has_key = __contains__+ def has_key(self, key: str | bytes) -> bool:+ warnings.warn(+ 'Headers.has_key() is deprecated, use the "in" operator instead.',+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return key in selfdef setdefault(self, key: str | bytes, def_val: Any = None) -> Any:return dict.setdefault(self, self._normkey(key), self._normvalue(def_val))@@ -51,6 +57,12 @@ class Headers(dict): # type: ignore[type-arg]def fromkeys( # type: ignore[override]cls, keys: Iterable[str | bytes], value: Any = None) -> Self:+ warnings.warn(+ "Headers.fromkeys() is deprecated, pass (key, value) pairs to"+ " Headers() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return cls((k, value) for k in keys)def pop(self, key: str | bytes, *args: Any) -> Any:@@ -152,11 +164,22 @@ class Headers(dict): # type: ignore[type-arg]return []def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:+ warnings.warn(+ "Headers.setlist() is deprecated, assign the list instead:"+ " headers[key] = values.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )self[key] = list_def setlistdefault(self, key: str | bytes, default_list: Iterable[_RawValue] = ()) -> Any:+ warnings.warn(+ "Headers.setlistdefault() is deprecated, use setdefault() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return self.setdefault(key, default_list)def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:@@ -177,10 +200,11 @@ class Headers(dict): # type: ignore[type-arg]return headers_dict_to_raw(self)def to_unicode_dict(self) -> CaseInsensitiveDict:- """Return headers as a CaseInsensitiveDict with str keys- and str values. Multiple values are joined with ','. Bytes that- cannot be decoded are replaced with U+FFFD.- """+ warnings.warn(+ "Headers.to_unicode_dict() is deprecated, use to_tuple_list() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return CaseInsensitiveDict((to_unicode(key, encoding=self.encoding, errors="replace"),diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex a479edfee..7e4932601 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -2,6 +2,7 @@ import copyimport pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Headers@@ -109,8 +110,15 @@ def test_setdefault() -> None:assert h.getlist("X-Forwarded-For") is olist+def test_has_key() -> None:+ h = Headers({"Content-Type": "text/html"})+ with pytest.warns(ScrapyDeprecationWarning, match="has_key"):+ assert h.has_key("content-type")++def test_fromkeys() -> None:- h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")+ with pytest.warns(ScrapyDeprecationWarning, match="fromkeys"):+ h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")assert h.getlist("Content-Type") == [b"value"]assert h.getlist("Content-Length") == [b"value"]@@ -161,14 +169,17 @@ def test_appendlist() -> None:def test_setlist() -> None:h1 = Headers({"header1": "value1"})assert h1.getlist("header1") == [b"value1"]- h1.setlist("header1", [b"value2", b"value3"])+ with pytest.warns(ScrapyDeprecationWarning, match="setlist"):+ h1.setlist("header1", [b"value2", b"value3"])assert h1.getlist("header1") == [b"value2", b"value3"]def test_setlistdefault() -> None:h1 = Headers({"header1": "value1"})- h1.setlistdefault("header1", ["value2", "value3"])- h1.setlistdefault("header2", ["value2", "value3"])+ with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):+ h1.setlistdefault("header1", ["value2", "value3"])+ with pytest.warns(ScrapyDeprecationWarning, match="setlistdefault"):+ h1.setlistdefault("header2", ["value2", "value3"])assert h1.getlist("header1") == [b"value1"]assert h1.getlist("header2") == [b"value2", b"value3"]@@ -187,7 +198,7 @@ def test_int_value() -> None:h1 = Headers({"hey": 5})h1["foo"] = 1h1.setdefault("bar", 2)- h1.setlist("buz", [1, "dos", 3])+ h1["buz"] = [1, "dos", 3]assert h1.getlist("foo") == [b"1"]assert h1.getlist("bar") == [b"2"]assert h1.getlist("buz") == [b"1", b"dos", b"3"]@@ -202,12 +213,14 @@ def test_invalid_value() -> None:with pytest.raises(TypeError, match="Unsupported value type"):Headers().setdefault("foo", object())with pytest.raises(TypeError, match="Unsupported value type"):- Headers().setlist("foo", [object()]) # type: ignore[list-item]+ Headers()["foo"] = [object()]def test_to_unicode_dict_undecodable() -> None:h = Headers({b"Public-Key-Pins": b'pin-sha256=\x94"a"', b"X-\xff": b"ok"})- assert h.to_unicode_dict() == {+ with pytest.warns(ScrapyDeprecationWarning, match="to_unicode_dict"):+ unicode_dict = h.to_unicode_dict()+ assert unicode_dict == {"Public-Key-Pins": 'pin-sha256=�"a"',"X-�": "ok",}diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.pyindex fea7f9f45..209bc30f3 100644--- a/tests/utils/bases/redirect.py+++ b/tests/utils/bases/redirect.py@@ -203,7 +203,7 @@ class TestRedirectBase(ABC):assert {**safe_headers,**cookie_header,- } == different_port_redirect_request.headers.to_unicode_dict()+ } == dict(different_port_redirect_request.headers.to_tuple_list())# A domain change drops both the Authorization and the Cookie header.external_response = self.get_response(original_request, "https://example.org/a")@@ -211,7 +211,7 @@ class TestRedirectBase(ABC):original_request, external_response)assert isinstance(external_redirect_request, Request)- assert safe_headers == external_redirect_request.headers.to_unicode_dict()+ assert safe_headers == dict(external_redirect_request.headers.to_tuple_list())# A scheme upgrade (http → https) drops the Authorization header# because the origin changes, but keeps the Cookie header because the@@ -224,7 +224,7 @@ class TestRedirectBase(ABC):assert {**safe_headers,**cookie_header,- } == upgrade_redirect_request.headers.to_unicode_dict()+ } == dict(upgrade_redirect_request.headers.to_tuple_list())# A scheme downgrade (https → http) drops the Authorization header# because the origin changes, and the Cookie header because its value@@ -240,7 +240,7 @@ class TestRedirectBase(ABC):original_request, downgrade_response)assert isinstance(downgrade_redirect_request, Request)- assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()+ assert safe_headers == dict(downgrade_redirect_request.headers.to_tuple_list())def _check_proxy_scenario(self,