Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate scrapy.utils.datatypes. (#8270)
Andrey Rakhmatullin6249564d362b
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex 809f6918d..c345e297c 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -9,7 +9,7 @@ from w3lib.url import is_urlfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.http import Request, Response-from scrapy.utils.datatypes import SequenceExclude+from scrapy.utils._datatypes import SequenceExcludefrom scrapy.utils.spider import DefaultSpider, spidercls_for_requestif TYPE_CHECKING:diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex e6f9e6289..b02ae4859 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Self, TypeAlias, castfrom w3lib.http import headers_dict_to_rawfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.datatypes import CaseInsensitiveDict+from scrapy.utils._datatypes import CaseInsensitiveDictfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex ae68918c4..6c37fef22 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -44,10 +44,10 @@ from scrapy.pipelines.media import (MediaPipeline,_MediaRequestFiltered,)+from scrapy.utils._datatypes import CaseInsensitiveDictfrom scrapy.utils._ftp import ftp_store_filefrom scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.boto import _get_max_pool_connections, is_botocore_available-from scrapy.utils.datatypes import CaseInsensitiveDictfrom scrapy.utils.defer import deferred_from_coro, ensure_awaitablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infodiff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 2b0f81ecd..484c6297c 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -13,9 +13,9 @@ from twisted.python.failure import Failurefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.request import NO_CALLBACK, Request+from scrapy.utils._datatypes import SequenceExcludefrom scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACKfrom scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (_process_pending_io,diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 6865eb463..b8dd25631 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -14,7 +14,7 @@ from twisted.internet.interfaces import ()from zope.interface.declarations import implementer, provider-from scrapy.utils.datatypes import LocalCache+from scrapy.utils._datatypes import LocalCacheif TYPE_CHECKING:from collections.abc import Sequencediff --git a/scrapy/utils/_datatypes.py b/scrapy/utils/_datatypes.pynew file mode 100644index 000000000..0f3e762d7--- /dev/null+++ b/scrapy/utils/_datatypes.py@@ -0,0 +1,130 @@+from __future__ import annotations++import collections+import contextlib+import weakref+from collections import OrderedDict+from typing import TYPE_CHECKING, Any, Self, TypeVar, cast++if TYPE_CHECKING:+ from collections.abc import Container+++_KT = TypeVar("_KT")+_VT = TypeVar("_VT")+++class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):+ """A dict-like structure that accepts strings or bytes+ as keys and allows case-insensitive lookups.+ """++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ self._keys: dict[str | bytes, Any] = {}+ super().__init__(*args, **kwargs)++ def __getitem__(self, key: str | bytes) -> Any:+ normalized_key = self._normkey(key)+ return super().__getitem__(self._keys[normalized_key.lower()])++ def __setitem__(self, key: str | bytes, value: Any) -> None:+ normalized_key = self._normkey(key)+ try:+ lower_key = self._keys[normalized_key.lower()]+ del self[lower_key]+ except KeyError:+ pass+ super().__setitem__(normalized_key, self._normvalue(value))+ self._keys[normalized_key.lower()] = normalized_key++ def __delitem__(self, key: str | bytes) -> None:+ normalized_key = self._normkey(key)+ stored_key = self._keys.pop(normalized_key.lower())+ super().__delitem__(stored_key)++ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]+ normalized_key = self._normkey(key)+ return normalized_key.lower() in self._keys++ def __repr__(self) -> str:+ return f"<{self.__class__.__name__}: {super().__repr__()}>"++ # UserDict.copy() shallow-copies the instance, which would share self._keys+ # between the copy and the original.+ def __copy__(self) -> Self:+ new = self.__class__()+ new.data = self.data.copy()+ new._keys = self._keys.copy()+ return new++ copy = __copy__++ # UserDict.__ior__ updates self.data directly, which would leave self._keys+ # out of date.+ def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]+ self.update(other)+ return self++ def _normkey(self, key: str | bytes) -> str | bytes:+ return key++ def _normvalue(self, value: Any) -> Any:+ return value+++class LocalCache(OrderedDict[_KT, _VT]):+ """Dictionary with a finite number of keys.++ Older items expires first.+ """++ def __init__(self, limit: int | None = None):+ super().__init__()+ self.limit: int | None = limit++ def __setitem__(self, key: _KT, value: _VT) -> None:+ if self.limit is not None:+ if self.limit == 0:+ return+ if key not in self:+ while len(self) >= self.limit:+ self.popitem(last=False)+ super().__setitem__(key, value)+++class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):+ """+ A weakref.WeakKeyDictionary implementation that uses LocalCache as its+ underlying data structure, making it ordered and capable of being size-limited.++ Useful for memoization, while avoiding keeping received+ arguments in memory only because of the cached references.++ Note: like LocalCache and unlike weakref.WeakKeyDictionary,+ it cannot be instantiated with an initial dictionary.+ """++ def __init__(self, limit: int | None = None):+ super().__init__()+ self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)++ def __setitem__(self, key: _KT, value: _VT | None) -> None:+ # if raised, key is not weak-referenceable, skip caching+ with contextlib.suppress(TypeError):+ super().__setitem__(key, value)++ def __getitem__(self, key: _KT) -> _VT | None:+ try:+ return cast("_VT", super().__getitem__(key))+ except (TypeError, KeyError):+ return None # key is either not weak-referenceable or not cached+++class SequenceExclude:+ """Object to test if an item is NOT within some sequence."""++ def __init__(self, seq: Container[object]):+ self.seq: Container[object] = seq++ def __contains__(self, item: object) -> bool:+ return item not in self.seqdiff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex f5f97cd88..7bdfea9ab 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -1,135 +1,23 @@-"""-This module contains data types used by Scrapy which are not included in the-Python Standard Library.-"""--from __future__ import annotations--import collections-import contextlib-import weakref-from collections import OrderedDict-from typing import TYPE_CHECKING, Any, Self, TypeVar, cast--if TYPE_CHECKING:- from collections.abc import Container---_KT = TypeVar("_KT")-_VT = TypeVar("_VT")---class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):- """A dict-like structure that accepts strings or bytes- as keys and allows case-insensitive lookups.- """-- def __init__(self, *args: Any, **kwargs: Any) -> None:- self._keys: dict[str | bytes, Any] = {}- super().__init__(*args, **kwargs)-- def __getitem__(self, key: str | bytes) -> Any:- normalized_key = self._normkey(key)- return super().__getitem__(self._keys[normalized_key.lower()])-- def __setitem__(self, key: str | bytes, value: Any) -> None:- normalized_key = self._normkey(key)- try:- lower_key = self._keys[normalized_key.lower()]- del self[lower_key]- except KeyError:- pass- super().__setitem__(normalized_key, self._normvalue(value))- self._keys[normalized_key.lower()] = normalized_key-- def __delitem__(self, key: str | bytes) -> None:- normalized_key = self._normkey(key)- stored_key = self._keys.pop(normalized_key.lower())- super().__delitem__(stored_key)-- def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]- normalized_key = self._normkey(key)- return normalized_key.lower() in self._keys-- def __repr__(self) -> str:- return f"<{self.__class__.__name__}: {super().__repr__()}>"-- # UserDict.copy() shallow-copies the instance, which would share self._keys- # between the copy and the original.- def __copy__(self) -> Self:- new = self.__class__()- new.data = self.data.copy()- new._keys = self._keys.copy()- return new-- copy = __copy__-- # UserDict.__ior__ updates self.data directly, which would leave self._keys- # out of date.- def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]- self.update(other)- return self-- def _normkey(self, key: str | bytes) -> str | bytes:- return key-- def _normvalue(self, value: Any) -> Any:- return value---class LocalCache(OrderedDict[_KT, _VT]):- """Dictionary with a finite number of keys.-- Older items expires first.- """-- def __init__(self, limit: int | None = None):- super().__init__()- self.limit: int | None = limit-- def __setitem__(self, key: _KT, value: _VT) -> None:- if self.limit is not None:- if self.limit == 0:- return- if key not in self:- while len(self) >= self.limit:- self.popitem(last=False)- super().__setitem__(key, value)---class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):- """- A weakref.WeakKeyDictionary implementation that uses LocalCache as its- underlying data structure, making it ordered and capable of being size-limited.-- Useful for memoization, while avoiding keeping received- arguments in memory only because of the cached references.-- Note: like LocalCache and unlike weakref.WeakKeyDictionary,- it cannot be instantiated with an initial dictionary.- """-- def __init__(self, limit: int | None = None):- super().__init__()- self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)-- def __setitem__(self, key: _KT, value: _VT | None) -> None:- # if raised, key is not weak-referenceable, skip caching- with contextlib.suppress(TypeError):- super().__setitem__(key, value)-- def __getitem__(self, key: _KT) -> _VT | None:- try:- return cast("_VT", super().__getitem__(key))- except (TypeError, KeyError):- return None # key is either not weak-referenceable or not cached---class SequenceExclude:- """Object to test if an item is NOT within some sequence."""-- def __init__(self, seq: Container[object]):- self.seq: Container[object] = seq-- def __contains__(self, item: object) -> bool:- return item not in self.seq+# pragma: no file cover+import warnings++from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._datatypes import (+ CaseInsensitiveDict,+ LocalCache,+ LocalWeakReferencedCache,+ SequenceExclude,+)++warnings.warn(+ "The scrapy.utils.datatypes module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)++__all__ = [+ "CaseInsensitiveDict",+ "LocalCache",+ "LocalWeakReferencedCache",+ "SequenceExclude",+]diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex d19299fc7..bbb3141c3 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overlfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.item import Item-from scrapy.utils.datatypes import LocalWeakReferencedCache+from scrapy.utils._datatypes import LocalWeakReferencedCacheif TYPE_CHECKING:from collections.abc import Callable, Iterable, Iteratordiff --git a/tests/ignores.txt b/tests/ignores.txtindex 6893acebc..4c6fa1dd3 100644--- a/tests/ignores.txt+++ b/tests/ignores.txt@@ -4,6 +4,7 @@ scrapy/interfaces.pyscrapy/mail.pyscrapy/utils/benchserver.pyscrapy/utils/console.py+scrapy/utils/datatypes.pyscrapy/utils/display.pyscrapy/utils/ftp.pyscrapy/utils/gz.pydiff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 55bef53b2..26096911d 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -7,7 +7,7 @@ from typing import Anyimport pytestfrom scrapy.http import Request-from scrapy.utils.datatypes import (+from scrapy.utils._datatypes import (CaseInsensitiveDict,LocalCache,LocalWeakReferencedCache,