Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate scrapy.utils.datatypes. (#8270)

Andrey Rakhmatullin6249564d362b
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 809f6918d..c345e297c 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -9,7 +9,7 @@ from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
-from scrapy.utils.datatypes import SequenceExclude
+from scrapy.utils._datatypes import SequenceExclude
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
if TYPE_CHECKING:
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index e6f9e6289..b02ae4859 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Self, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.datatypes import CaseInsensitiveDict
+from scrapy.utils._datatypes import CaseInsensitiveDict
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ae68918c4..6c37fef22 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -44,10 +44,10 @@ from scrapy.pipelines.media import (
MediaPipeline,
_MediaRequestFiltered,
)
+from scrapy.utils._datatypes import CaseInsensitiveDict
from scrapy.utils._ftp import ftp_store_file
from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
-from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 2b0f81ecd..484c6297c 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -13,9 +13,9 @@ from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import NO_CALLBACK, Request
+from scrapy.utils._datatypes import SequenceExclude
from scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACK
from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
_process_pending_io,
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 6865eb463..b8dd25631 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -14,7 +14,7 @@ from twisted.internet.interfaces import (
)
from zope.interface.declarations import implementer, provider
-from scrapy.utils.datatypes import LocalCache
+from scrapy.utils._datatypes import LocalCache
if TYPE_CHECKING:
from collections.abc import Sequence
diff --git a/scrapy/utils/_datatypes.py b/scrapy/utils/_datatypes.py
new file mode 100644
index 000000000..0f3e762d7
--- /dev/null
+++ b/scrapy/utils/_datatypes.py
@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import collections
+import contextlib
+import weakref
+from collections import OrderedDict
+from typing import TYPE_CHECKING, Any, Self, TypeVar, cast
+
+if TYPE_CHECKING:
+ from collections.abc import Container
+
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
+
+
+class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
+ """A dict-like structure that accepts strings or bytes
+ as keys and allows case-insensitive lookups.
+ """
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ self._keys: dict[str | bytes, Any] = {}
+ super().__init__(*args, **kwargs)
+
+ def __getitem__(self, key: str | bytes) -> Any:
+ normalized_key = self._normkey(key)
+ return super().__getitem__(self._keys[normalized_key.lower()])
+
+ def __setitem__(self, key: str | bytes, value: Any) -> None:
+ normalized_key = self._normkey(key)
+ try:
+ lower_key = self._keys[normalized_key.lower()]
+ del self[lower_key]
+ except KeyError:
+ pass
+ super().__setitem__(normalized_key, self._normvalue(value))
+ self._keys[normalized_key.lower()] = normalized_key
+
+ def __delitem__(self, key: str | bytes) -> None:
+ normalized_key = self._normkey(key)
+ stored_key = self._keys.pop(normalized_key.lower())
+ super().__delitem__(stored_key)
+
+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
+ normalized_key = self._normkey(key)
+ return normalized_key.lower() in self._keys
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}: {super().__repr__()}>"
+
+ # UserDict.copy() shallow-copies the instance, which would share self._keys
+ # between the copy and the original.
+ def __copy__(self) -> Self:
+ new = self.__class__()
+ new.data = self.data.copy()
+ new._keys = self._keys.copy()
+ return new
+
+ copy = __copy__
+
+ # UserDict.__ior__ updates self.data directly, which would leave self._keys
+ # out of date.
+ def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]
+ self.update(other)
+ return self
+
+ def _normkey(self, key: str | bytes) -> str | bytes:
+ return key
+
+ def _normvalue(self, value: Any) -> Any:
+ return value
+
+
+class LocalCache(OrderedDict[_KT, _VT]):
+ """Dictionary with a finite number of keys.
+
+ Older items expires first.
+ """
+
+ def __init__(self, limit: int | None = None):
+ super().__init__()
+ self.limit: int | None = limit
+
+ def __setitem__(self, key: _KT, value: _VT) -> None:
+ if self.limit is not None:
+ if self.limit == 0:
+ return
+ if key not in self:
+ while len(self) >= self.limit:
+ self.popitem(last=False)
+ super().__setitem__(key, value)
+
+
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
+ """
+ A weakref.WeakKeyDictionary implementation that uses LocalCache as its
+ underlying data structure, making it ordered and capable of being size-limited.
+
+ Useful for memoization, while avoiding keeping received
+ arguments in memory only because of the cached references.
+
+ Note: like LocalCache and unlike weakref.WeakKeyDictionary,
+ it cannot be instantiated with an initial dictionary.
+ """
+
+ def __init__(self, limit: int | None = None):
+ super().__init__()
+ self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
+
+ def __setitem__(self, key: _KT, value: _VT | None) -> None:
+ # if raised, key is not weak-referenceable, skip caching
+ with contextlib.suppress(TypeError):
+ super().__setitem__(key, value)
+
+ def __getitem__(self, key: _KT) -> _VT | None:
+ try:
+ return cast("_VT", super().__getitem__(key))
+ except (TypeError, KeyError):
+ return None # key is either not weak-referenceable or not cached
+
+
+class SequenceExclude:
+ """Object to test if an item is NOT within some sequence."""
+
+ def __init__(self, seq: Container[object]):
+ self.seq: Container[object] = seq
+
+ def __contains__(self, item: object) -> bool:
+ return item not in self.seq
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index f5f97cd88..7bdfea9ab 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -1,135 +1,23 @@
-"""
-This module contains data types used by Scrapy which are not included in the
-Python Standard Library.
-"""
-
-from __future__ import annotations
-
-import collections
-import contextlib
-import weakref
-from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Self, TypeVar, cast
-
-if TYPE_CHECKING:
- from collections.abc import Container
-
-
-_KT = TypeVar("_KT")
-_VT = TypeVar("_VT")
-
-
-class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
- """A dict-like structure that accepts strings or bytes
- as keys and allows case-insensitive lookups.
- """
-
- def __init__(self, *args: Any, **kwargs: Any) -> None:
- self._keys: dict[str | bytes, Any] = {}
- super().__init__(*args, **kwargs)
-
- def __getitem__(self, key: str | bytes) -> Any:
- normalized_key = self._normkey(key)
- return super().__getitem__(self._keys[normalized_key.lower()])
-
- def __setitem__(self, key: str | bytes, value: Any) -> None:
- normalized_key = self._normkey(key)
- try:
- lower_key = self._keys[normalized_key.lower()]
- del self[lower_key]
- except KeyError:
- pass
- super().__setitem__(normalized_key, self._normvalue(value))
- self._keys[normalized_key.lower()] = normalized_key
-
- def __delitem__(self, key: str | bytes) -> None:
- normalized_key = self._normkey(key)
- stored_key = self._keys.pop(normalized_key.lower())
- super().__delitem__(stored_key)
-
- def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
- normalized_key = self._normkey(key)
- return normalized_key.lower() in self._keys
-
- def __repr__(self) -> str:
- return f"<{self.__class__.__name__}: {super().__repr__()}>"
-
- # UserDict.copy() shallow-copies the instance, which would share self._keys
- # between the copy and the original.
- def __copy__(self) -> Self:
- new = self.__class__()
- new.data = self.data.copy()
- new._keys = self._keys.copy()
- return new
-
- copy = __copy__
-
- # UserDict.__ior__ updates self.data directly, which would leave self._keys
- # out of date.
- def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]
- self.update(other)
- return self
-
- def _normkey(self, key: str | bytes) -> str | bytes:
- return key
-
- def _normvalue(self, value: Any) -> Any:
- return value
-
-
-class LocalCache(OrderedDict[_KT, _VT]):
- """Dictionary with a finite number of keys.
-
- Older items expires first.
- """
-
- def __init__(self, limit: int | None = None):
- super().__init__()
- self.limit: int | None = limit
-
- def __setitem__(self, key: _KT, value: _VT) -> None:
- if self.limit is not None:
- if self.limit == 0:
- return
- if key not in self:
- while len(self) >= self.limit:
- self.popitem(last=False)
- super().__setitem__(key, value)
-
-
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
- """
- A weakref.WeakKeyDictionary implementation that uses LocalCache as its
- underlying data structure, making it ordered and capable of being size-limited.
-
- Useful for memoization, while avoiding keeping received
- arguments in memory only because of the cached references.
-
- Note: like LocalCache and unlike weakref.WeakKeyDictionary,
- it cannot be instantiated with an initial dictionary.
- """
-
- def __init__(self, limit: int | None = None):
- super().__init__()
- self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
-
- def __setitem__(self, key: _KT, value: _VT | None) -> None:
- # if raised, key is not weak-referenceable, skip caching
- with contextlib.suppress(TypeError):
- super().__setitem__(key, value)
-
- def __getitem__(self, key: _KT) -> _VT | None:
- try:
- return cast("_VT", super().__getitem__(key))
- except (TypeError, KeyError):
- return None # key is either not weak-referenceable or not cached
-
-
-class SequenceExclude:
- """Object to test if an item is NOT within some sequence."""
-
- def __init__(self, seq: Container[object]):
- self.seq: Container[object] = seq
-
- def __contains__(self, item: object) -> bool:
- return item not in self.seq
+# pragma: no file cover
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._datatypes import (
+ CaseInsensitiveDict,
+ LocalCache,
+ LocalWeakReferencedCache,
+ SequenceExclude,
+)
+
+warnings.warn(
+ "The scrapy.utils.datatypes module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "CaseInsensitiveDict",
+ "LocalCache",
+ "LocalWeakReferencedCache",
+ "SequenceExclude",
+]
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index d19299fc7..bbb3141c3 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overl
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
-from scrapy.utils.datatypes import LocalWeakReferencedCache
+from scrapy.utils._datatypes import LocalWeakReferencedCache
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator
diff --git a/tests/ignores.txt b/tests/ignores.txt
index 6893acebc..4c6fa1dd3 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -4,6 +4,7 @@ scrapy/interfaces.py
scrapy/mail.py
scrapy/utils/benchserver.py
scrapy/utils/console.py
+scrapy/utils/datatypes.py
scrapy/utils/display.py
scrapy/utils/ftp.py
scrapy/utils/gz.py
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 55bef53b2..26096911d 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -7,7 +7,7 @@ from typing import Any
import pytest
from scrapy.http import Request
-from scrapy.utils.datatypes import (
+from scrapy.utils._datatypes import (
CaseInsensitiveDict,
LocalCache,
LocalWeakReferencedCache,