Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CaseInsensitiveDict: preserve original keys (only lookups are key-insensitive)

Eugenio Lacuestabbeed6ae8fd9
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index ca6089e0f..1d56811f0 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -79,35 +79,45 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict,):
"""A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
"""
+ def __init__(self, *args, **kwargs) -> None:
+ self._keys: dict = {}
+ super().__init__(*args, **kwargs)
+
def __getitem__(self, key: AnyStr) -> Any:
- return super().__getitem__(self.normkey(key))
+ normalized_key = self.normkey(key)
+ return super().__getitem__(self._keys[normalized_key.lower()])
def __setitem__(self, key: AnyStr, value: Any) -> None:
- super().__setitem__(self.normkey(key), self.normvalue(value))
+ normalized_key = self.normkey(key)
+ if normalized_key.lower() in self._keys:
+ del self[self._keys[normalized_key.lower()]]
+ super().__setitem__(normalized_key, self.normvalue(value))
+ self._keys[normalized_key.lower()] = normalized_key
def __delitem__(self, key: AnyStr) -> None:
- super().__delitem__(self.normkey(key))
+ normalized_key = self.normkey(key)
+ stored_key = self._keys.pop(normalized_key.lower())
+ super().__delitem__(stored_key)
def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
- return super().__contains__(self.normkey(key))
+ normalized_key = self.normkey(key)
+ return normalized_key.lower() in self._keys
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}: {super().__repr__()}>"
def normkey(self, key: AnyStr) -> AnyStr:
- """Method to normalize dictionary key access"""
- return key.lower()
+ return key
def normvalue(self, value: Any) -> Any:
- """Method to normalize values prior to be set"""
return value
- def __repr__(self) -> str:
- return f"<{self.__class__.__name__}: {super().__repr__()}>"
-
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index c033cd537..5faaabe81 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,4 +1,5 @@
import copy
+from typing import Iterator
import unittest
import warnings
from collections.abc import Mapping, MutableMapping
@@ -191,8 +192,16 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
dict_class = CaseInsensitiveDict
def test_repr(self):
- d = self.dict_class({"foo": "bar"})
- self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+ d1 = self.dict_class({"foo": "bar"})
+ self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+ d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+ self.assertEqual(repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>")
+
+ def test_iter(self):
+ d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+ iterkeys = iter(d)
+ self.assertIsInstance(iterkeys, Iterator)
+ self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):