Scrapy, a fast high-level web crawling & scraping framework for Python.
Underscore CaseInsensitiveDict normkey/normvalue
Eugenio Lacuesta1c031b8a8dd7
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex f45e1c9b8..807a95504 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -80,9 +80,8 @@ class CaselessDict(dict):class CaseInsensitiveDict(collections.UserDict):- """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.-- It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.+ """A dict-like structure that accepts strings or bytes+ as keys and allows case-insensitive lookups."""def __init__(self, *args, **kwargs) -> None:@@ -90,32 +89,32 @@ class CaseInsensitiveDict(collections.UserDict):super().__init__(*args, **kwargs)def __getitem__(self, key: AnyStr) -> Any:- normalized_key = self.normkey(key)+ normalized_key = self._normkey(key)return super().__getitem__(self._keys[normalized_key.lower()])def __setitem__(self, key: AnyStr, value: Any) -> None:- normalized_key = self.normkey(key)+ normalized_key = self._normkey(key)if normalized_key.lower() in self._keys:del self[self._keys[normalized_key.lower()]]- super().__setitem__(normalized_key, self.normvalue(value))+ super().__setitem__(normalized_key, self._normvalue(value))self._keys[normalized_key.lower()] = normalized_keydef __delitem__(self, key: AnyStr) -> None:- normalized_key = self.normkey(key)+ normalized_key = self._normkey(key)stored_key = self._keys.pop(normalized_key.lower())super().__delitem__(stored_key)def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]- normalized_key = self.normkey(key)+ normalized_key = self._normkey(key)return normalized_key.lower() in self._keysdef __repr__(self) -> str:return f"<{self.__class__.__name__}: {super().__repr__()}>"- def normkey(self, key: AnyStr) -> AnyStr:+ def _normkey(self, key: AnyStr) -> AnyStr:return key- def normvalue(self, value: Any) -> Any:+ def _normvalue(self, value: Any) -> Any:return valuediff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 5faaabe81..0a724f237 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -1,8 +1,8 @@import copy-from typing import Iteratorimport unittestimport warningsfrom collections.abc import Mapping, MutableMapping+from typing import Iteratorfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request@@ -144,19 +144,23 @@ class CaseInsensitiveDictMixin:def test_normkey(self):class MyDict(self.dict_class):- def normkey(self, key):+ def _normkey(self, key):return key.title()+ normkey = _normkey # deprecated CaselessDict class+d = MyDict()d['key-one'] = 2self.assertEqual(list(d.keys()), ['Key-One'])def test_normvalue(self):class MyDict(self.dict_class):- def normvalue(self, value):+ def _normvalue(self, value):if value is not None:return value + 1+ normvalue = _normvalue # deprecated CaselessDict class+d = MyDict({'key': 1})self.assertEqual(d['key'], 2)self.assertEqual(d.get('key'), 2)