Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove deprecated CaselessDict and md5sum() (#8023)

Adriandf64fe32f614
diff --git a/docs/news.rst b/docs/news.rst
index 38624948d..50f845e4c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -2810,6 +2810,9 @@ Deprecations
instead call the builder callable directly.
(:gh:`6540`)
+- ``scrapy.utils.misc.md5sum()`` is now deprecated.
+ (:gh:`6264`)
+
New features
~~~~~~~~~~~~
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index b55ef6191..dead57cdb 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
-from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
@@ -18,9 +18,7 @@ if TYPE_CHECKING:
_RawValue: TypeAlias = bytes | str | int
-# isn't fully compatible typing-wise with either dict or CaselessDict,
-# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146
-class Headers(CaselessDict):
+class Headers(dict): # type: ignore[type-arg]
"""Case insensitive http headers dictionary"""
def __init__(
@@ -32,7 +30,32 @@ class Headers(CaselessDict):
encoding: str = "utf-8",
):
self.encoding: str = encoding
- super().__init__(seq)
+ super().__init__()
+ if seq:
+ self.update(seq)
+
+ def __setitem__(self, key: str | bytes, value: Any) -> None:
+ dict.__setitem__(self, self.normkey(key), self.normvalue(value))
+
+ def __delitem__(self, key: str | bytes) -> None:
+ dict.__delitem__(self, self.normkey(key))
+
+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
+ return dict.__contains__(self, self.normkey(key))
+
+ has_key = __contains__
+
+ def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
+ return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
+
+ @classmethod
+ def fromkeys( # type: ignore[override]
+ cls, keys: Iterable[str | bytes], value: Any = None
+ ) -> Self:
+ return cls((k, value) for k in keys)
+
+ def pop(self, key: str | bytes, *args: Any) -> Any:
+ return dict.pop(self, self.normkey(key), *args)
def update( # type: ignore[override]
self,
@@ -44,7 +67,7 @@ class Headers(CaselessDict):
iseq: dict[bytes, list[bytes]] = {}
for k, v in seq:
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
- super().update(iseq)
+ dict.update(self, iseq)
def normkey(self, key: str | bytes) -> bytes:
"""Normalize key to bytes"""
@@ -75,19 +98,22 @@ class Headers(CaselessDict):
def __getitem__(self, key: str | bytes) -> bytes | None:
try:
- return cast("list[bytes]", super().__getitem__(key))[-1]
+ return cast("list[bytes]", dict.__getitem__(self, self.normkey(key)))[-1]
except IndexError:
return None
def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
try:
- return cast("list[bytes]", super().get(key, def_val))[-1]
+ return cast(
+ "list[bytes]",
+ dict.get(self, self.normkey(key), self.normvalue(def_val)),
+ )[-1]
except IndexError:
return None
def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
try:
- return cast("list[bytes]", super().__getitem__(key))
+ return cast("list[bytes]", dict.__getitem__(self, self.normkey(key)))
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 9a945c61c..47d1de096 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,16 +7,12 @@ from __future__ import annotations
import collections
import contextlib
-import warnings
import weakref
from collections import OrderedDict
-from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, TypeVar, cast
-from scrapy.exceptions import ScrapyDeprecationWarning
-
if TYPE_CHECKING:
- from collections.abc import Container, Iterable
+ from collections.abc import Container
# typing.Self requires Python 3.11
from typing_extensions import Self
@@ -26,85 +22,6 @@ _KT = TypeVar("_KT")
_VT = TypeVar("_VT")
-class CaselessDict(dict): # type: ignore[type-arg]
- __slots__ = ()
-
- def __new__(cls, *args: Any, **kwargs: Any) -> Self:
- # circular import
- from scrapy.http.headers import Headers # noqa: PLC0415
-
- if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
- warnings.warn(
- "scrapy.utils.datatypes.CaselessDict is deprecated,"
- " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return super().__new__(cls, *args, **kwargs)
-
- def __init__(
- self,
- seq: Mapping[str, Any]
- | Mapping[bytes, Any]
- | Iterable[tuple[str | bytes, Any]]
- | None = None,
- ):
- super().__init__()
- if seq:
- self.update(seq)
-
- def __getitem__(self, key: str | bytes) -> Any:
- return dict.__getitem__(self, self.normkey(key))
-
- def __setitem__(self, key: str | bytes, value: Any) -> None:
- dict.__setitem__(self, self.normkey(key), self.normvalue(value))
-
- def __delitem__(self, key: str | bytes) -> None:
- dict.__delitem__(self, self.normkey(key))
-
- def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
- return dict.__contains__(self, self.normkey(key))
-
- has_key = __contains__
-
- def __copy__(self) -> Self:
- return self.__class__(self)
-
- copy = __copy__
-
- def normkey(self, key: str | bytes) -> str | bytes:
- """Method to normalize dictionary key access"""
- return key.lower()
-
- def normvalue(self, value: Any) -> Any:
- """Method to normalize values prior to be set"""
- return value
-
- def get(self, key: str | bytes, def_val: Any = None) -> Any:
- return dict.get(self, self.normkey(key), self.normvalue(def_val))
-
- def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
- return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
-
- # doesn't fully implement MutableMapping.update()
- def update( # type: ignore[override]
- self,
- seq: Mapping[str, Any]
- | Mapping[bytes, Any]
- | Iterable[tuple[str | bytes, Any]],
- ) -> None:
- seq = seq.items() if isinstance(seq, Mapping) else seq
- iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
- super().update(iseq)
-
- @classmethod
- def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]
- return cls((k, value) for k in keys)
-
- def pop(self, key: str | bytes, *args: Any) -> Any:
- return dict.pop(self, self.normkey(key), *args)
-
-
class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
"""A dict-like structure that accepts strings or bytes
as keys and allows case-insensitive lookups.
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 470742f1f..7956a99df 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import ast
-import hashlib
import inspect
import os
import re
@@ -13,7 +12,7 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
-from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload
+from typing import TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
@@ -140,26 +139,6 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover
return list(walk_modules_iter(path))
-def md5sum(file: IO[bytes]) -> str: # pragma: no cover
- """Calculate the md5 checksum of a file-like object without reading its
- whole content in memory."""
- warnings.warn(
- (
- "The scrapy.utils.misc.md5sum function is deprecated and will be "
- "removed in a future version of Scrapy."
- ),
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- m = hashlib.md5() # noqa: S324
- while True:
- d = file.read(8096)
- if not d:
- break
- m.update(d)
- return m.hexdigest()
-
-
def rel_has_nofollow(rel: str | None) -> bool:
"""Return True if link rel attribute has nofollow type"""
return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 561546711..a35f51cd8 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -88,6 +88,12 @@ def test_setdefault() -> None:
assert h.getlist("X-Forwarded-For") is olist
+def test_fromkeys() -> None:
+ h = Headers.fromkeys(("Content-Type", "Content-Length"), "value")
+ assert h.getlist("Content-Type") == [b"value"]
+ assert h.getlist("Content-Length") == [b"value"]
+
+
def test_iterables() -> None:
idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index af203ca61..555a0bd50 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,31 +1,23 @@
from __future__ import annotations
import copy
-from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
-from typing import Any, Generic, TypeVar
+from typing import Any
import pytest
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.datatypes import (
CaseInsensitiveDict,
- CaselessDict,
LocalCache,
LocalWeakReferencedCache,
SequenceExclude,
)
from scrapy.utils.python import garbage_collect
-_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
-
-class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
- @property
- @abstractmethod
- def dict_class(self) -> type[_DictT]:
- raise NotImplementedError
+class TestCaseInsensitiveDict:
+ dict_class = CaseInsensitiveDict
def test_init_dict(self):
seq = {"red": 1, "black": 3}
@@ -101,16 +93,14 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
with pytest.raises(KeyError):
d["key_lower"]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_getdefault(self):
- d = CaselessDict()
+ d = self.dict_class()
assert d.get("c", 5) == 5
d["c"] = 10
assert d.get("c", 5) == 10
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_setdefault(self):
- d = CaselessDict({"a": 1, "b": 2})
+ d = self.dict_class({"a": 1, "b": 2})
r = d.setdefault("A", 5)
assert r == 1
@@ -157,8 +147,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
def _normkey(self, key):
return key.title()
- normkey = _normkey # deprecated CaselessDict class
-
d = MyDict()
d["key-one"] = 2
assert list(d.keys()) == ["Key-One"]
@@ -170,8 +158,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
return value + 1
return None
- normvalue = _normvalue # deprecated CaselessDict class
-
d = MyDict({"key": 1})
assert d["key"] == 2
assert d.get("key") == 2
@@ -217,10 +203,6 @@ class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
assert "header3" not in h1
assert dict(h1) == {"header1": "value1", "header2": "value2"}
-
-class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
- dict_class = CaseInsensitiveDict
-
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
assert repr(d1) == "<CaseInsensitiveDict: {'foo': 'bar'}>"
@@ -256,18 +238,6 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
assert d["HeAdEr1"] == "value2"
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
- dict_class = CaselessDict
-
- def test_deprecation_message(self):
- with pytest.warns(
- ScrapyDeprecationWarning,
- match=r"scrapy.utils.datatypes.CaselessDict is deprecated",
- ):
- self.dict_class({"foo": "bar"})
-
-
class TestSequenceExclude:
def test_list(self):
seq = [1, 2, 3]