Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove deprecated CaselessDict and md5sum() (#8023)
Adriandf64fe32f614
diff --git a/docs/news.rst b/docs/news.rstindex 38624948d..50f845e4c 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -2810,6 +2810,9 @@ Deprecationsinstead call the builder callable directly.(:gh:`6540`)+- ``scrapy.utils.misc.md5sum()`` is now deprecated.+ (:gh:`6264`)+New features~~~~~~~~~~~~diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex b55ef6191..dead57cdb 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, TypeAlias, castfrom w3lib.http import headers_dict_to_raw-from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict+from scrapy.utils.datatypes import CaseInsensitiveDictfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:@@ -18,9 +18,7 @@ if TYPE_CHECKING:_RawValue: TypeAlias = bytes | str | int-# isn't fully compatible typing-wise with either dict or CaselessDict,-# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146-class Headers(CaselessDict):+class Headers(dict): # type: ignore[type-arg]"""Case insensitive http headers dictionary"""def __init__(@@ -32,7 +30,32 @@ class Headers(CaselessDict):encoding: str = "utf-8",):self.encoding: str = encoding- super().__init__(seq)+ super().__init__()+ if seq:+ self.update(seq)++ def __setitem__(self, key: str | bytes, value: Any) -> None:+ dict.__setitem__(self, self.normkey(key), self.normvalue(value))++ def __delitem__(self, key: str | bytes) -> None:+ dict.__delitem__(self, self.normkey(key))++ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]+ return dict.__contains__(self, self.normkey(key))++ has_key = __contains__++ def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:+ return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))++ @classmethod+ def fromkeys( # type: ignore[override]+ cls, keys: Iterable[str | bytes], value: Any = None+ ) -> Self:+ return cls((k, value) for k in keys)++ def pop(self, key: str | bytes, *args: Any) -> Any:+ return dict.pop(self, self.normkey(key), *args)def update( # type: ignore[override]self,@@ -44,7 +67,7 @@ class Headers(CaselessDict):iseq: dict[bytes, list[bytes]] = {}for k, v in seq:iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))- super().update(iseq)+ dict.update(self, iseq)def normkey(self, key: str | bytes) -> bytes:"""Normalize key to bytes"""@@ -75,19 +98,22 @@ class Headers(CaselessDict):def __getitem__(self, key: str | bytes) -> bytes | None:try:- return cast("list[bytes]", super().__getitem__(key))[-1]+ return cast("list[bytes]", dict.__getitem__(self, self.normkey(key)))[-1]except IndexError:return Nonedef get(self, key: str | bytes, def_val: Any = None) -> bytes | None:try:- return cast("list[bytes]", super().get(key, def_val))[-1]+ return cast(+ "list[bytes]",+ dict.get(self, self.normkey(key), self.normvalue(def_val)),+ )[-1]except IndexError:return Nonedef getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:try:- return cast("list[bytes]", super().__getitem__(key))+ return cast("list[bytes]", dict.__getitem__(self, self.normkey(key)))except KeyError:if def_val is not None:return self.normvalue(def_val)diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 9a945c61c..47d1de096 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -7,16 +7,12 @@ from __future__ import annotationsimport collectionsimport contextlib-import warningsimport weakreffrom collections import OrderedDict-from collections.abc import Mappingfrom typing import TYPE_CHECKING, Any, TypeVar, cast-from scrapy.exceptions import ScrapyDeprecationWarning-if TYPE_CHECKING:- from collections.abc import Container, Iterable+ from collections.abc import Container# typing.Self requires Python 3.11from typing_extensions import Self@@ -26,85 +22,6 @@ _KT = TypeVar("_KT")_VT = TypeVar("_VT")-class CaselessDict(dict): # type: ignore[type-arg]- __slots__ = ()-- def __new__(cls, *args: Any, **kwargs: Any) -> Self:- # circular import- from scrapy.http.headers import Headers # noqa: PLC0415-- if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):- warnings.warn(- "scrapy.utils.datatypes.CaselessDict is deprecated,"- " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return super().__new__(cls, *args, **kwargs)-- def __init__(- self,- seq: Mapping[str, Any]- | Mapping[bytes, Any]- | Iterable[tuple[str | bytes, Any]]- | None = None,- ):- super().__init__()- if seq:- self.update(seq)-- def __getitem__(self, key: str | bytes) -> Any:- return dict.__getitem__(self, self.normkey(key))-- def __setitem__(self, key: str | bytes, value: Any) -> None:- dict.__setitem__(self, self.normkey(key), self.normvalue(value))-- def __delitem__(self, key: str | bytes) -> None:- dict.__delitem__(self, self.normkey(key))-- def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]- return dict.__contains__(self, self.normkey(key))-- has_key = __contains__-- def __copy__(self) -> Self:- return self.__class__(self)-- copy = __copy__-- def normkey(self, key: str | bytes) -> str | bytes:- """Method to normalize dictionary key access"""- return key.lower()-- def normvalue(self, value: Any) -> Any:- """Method to normalize values prior to be set"""- return value-- def get(self, key: str | bytes, def_val: Any = None) -> Any:- return dict.get(self, self.normkey(key), self.normvalue(def_val))-- def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:- return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))-- # doesn't fully implement MutableMapping.update()- def update( # type: ignore[override]- self,- seq: Mapping[str, Any]- | Mapping[bytes, Any]- | Iterable[tuple[str | bytes, Any]],- ) -> None:- seq = seq.items() if isinstance(seq, Mapping) else seq- iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)- super().update(iseq)-- @classmethod- def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]- return cls((k, value) for k in keys)-- def pop(self, key: str | bytes, *args: Any) -> Any:- return dict.pop(self, self.normkey(key), *args)--class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):"""A dict-like structure that accepts strings or bytesas keys and allows case-insensitive lookups.diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 470742f1f..7956a99df 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -3,7 +3,6 @@from __future__ import annotationsimport ast-import hashlibimport inspectimport osimport re@@ -13,7 +12,7 @@ from contextlib import contextmanagerfrom functools import partialfrom importlib import import_modulefrom pkgutil import iter_modules-from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload+from typing import TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overloadfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.item import Item@@ -140,26 +139,6 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no coverreturn list(walk_modules_iter(path))-def md5sum(file: IO[bytes]) -> str: # pragma: no cover- """Calculate the md5 checksum of a file-like object without reading its- whole content in memory."""- warnings.warn(- (- "The scrapy.utils.misc.md5sum function is deprecated and will be "- "removed in a future version of Scrapy."- ),- ScrapyDeprecationWarning,- stacklevel=2,- )- m = hashlib.md5() # noqa: S324- while True:- d = file.read(8096)- if not d:- break- m.update(d)- return m.hexdigest()--def rel_has_nofollow(rel: str | None) -> bool:"""Return True if link rel attribute has nofollow type"""return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 561546711..a35f51cd8 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -88,6 +88,12 @@ def test_setdefault() -> None:assert h.getlist("X-Forwarded-For") is olist+def test_fromkeys() -> None:+ h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")+ assert h.getlist("Content-Type") == [b"value"]+ assert h.getlist("Content-Length") == [b"value"]++def test_iterables() -> None:idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex af203ca61..555a0bd50 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -1,31 +1,23 @@from __future__ import annotationsimport copy-from abc import ABC, abstractmethodfrom collections.abc import Iterator, Mapping, MutableMapping-from typing import Any, Generic, TypeVar+from typing import Anyimport pytest-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.utils.datatypes import (CaseInsensitiveDict,- CaselessDict,LocalCache,LocalWeakReferencedCache,SequenceExclude,)from scrapy.utils.python import garbage_collect-_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")--class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):- @property- @abstractmethod- def dict_class(self) -> type[_DictT]:- raise NotImplementedError+class TestCaseInsensitiveDict:+ dict_class = CaseInsensitiveDictdef test_init_dict(self):seq = {"red": 1, "black": 3}@@ -101,16 +93,14 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):with pytest.raises(KeyError):d["key_lower"]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_getdefault(self):- d = CaselessDict()+ d = self.dict_class()assert d.get("c", 5) == 5d["c"] = 10assert d.get("c", 5) == 10- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_setdefault(self):- d = CaselessDict({"a": 1, "b": 2})+ d = self.dict_class({"a": 1, "b": 2})r = d.setdefault("A", 5)assert r == 1@@ -157,8 +147,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):def _normkey(self, key):return key.title()- normkey = _normkey # deprecated CaselessDict class-d = MyDict()d["key-one"] = 2assert list(d.keys()) == ["Key-One"]@@ -170,8 +158,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):return value + 1return None- normvalue = _normvalue # deprecated CaselessDict class-d = MyDict({"key": 1})assert d["key"] == 2assert d.get("key") == 2@@ -217,10 +203,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):assert "header3" not in h1assert dict(h1) == {"header1": "value1", "header2": "value2"}--class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):- dict_class = CaseInsensitiveDict-def test_repr(self):d1 = self.dict_class({"foo": "bar"})assert repr(d1) == "<CaseInsensitiveDict: {'foo': 'bar'}>"@@ -256,18 +238,6 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):assert d["HeAdEr1"] == "value2"[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):- dict_class = CaselessDict-- def test_deprecation_message(self):- with pytest.warns(- ScrapyDeprecationWarning,- match=r"scrapy.utils.datatypes.CaselessDict is deprecated",- ):- self.dict_class({"foo": "bar"})--class TestSequenceExclude:def test_list(self):seq = [1, 2, 3]