Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix CaseInsensitiveDict.copy() sharing state with the original (#7783)

Shadow_Lu59ebb26e60ea
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index c020ff4b9..e761a2474 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -132,6 +132,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
+ # UserDict.copy() shallow-copies the instance, which would share self._keys
+ # between the copy and the original.
+ def __copy__(self) -> Self:
+ new = self.__class__()
+ new.data = self.data.copy()
+ new._keys = self._keys.copy()
+ return new
+
+ copy = __copy__
+
+ # UserDict.__ior__ updates self.data directly, which would leave self._keys
+ # out of date.
+ def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]
+ self.update(other)
+ return self
+
def _normkey(self, key: str | bytes) -> str | bytes:
return key
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index fe1f60c7d..f43d20e69 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -204,6 +204,15 @@ class TestCaseInsensitiveDictBase(ABC):
assert h1.get("header1") == h3.get("header1")
assert h1.get("header1") == h3.get("HEADER1")
+ def test_copy_is_independent(self):
+ h1 = self.dict_class({"header1": "value1", "header2": "value2"})
+ for h2 in (copy.copy(h1), h1.copy()):
+ del h2["header1"]
+ h2["header3"] = "value3"
+ assert "header1" in h1
+ assert "header3" not in h1
+ assert dict(h1) == {"header1": "value1", "header2": "value2"}
+
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict # type: ignore[assignment]
@@ -220,6 +229,28 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
assert isinstance(iterkeys, Iterator)
assert list(iterkeys) == ["AsDf", "FoO"]
+ def test_copy_keeps_values(self):
+ class MyDict(self.dict_class):
+ def _normvalue(self, value):
+ return value + 1
+
+ d = MyDict({"key": 1})
+ for copied in (copy.copy(d), d.copy()):
+ assert copied["key"] == 2
+
+ def test_ior(self):
+ d = self.dict_class({"header1": "value1"})
+ d |= {"HEADER1": "value2", "header2": "value3"}
+ assert len(d) == 2
+ assert d["HeAdEr1"] == "value2"
+ assert d["HeAdEr2"] == "value3"
+
+ def test_ior_mapping(self):
+ d = self.dict_class({"header1": "value1"})
+ d |= self.dict_class({"HEADER1": "value2"})
+ assert len(d) == 1
+ assert d["HeAdEr1"] == "value2"
+
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestCaselessDict(TestCaseInsensitiveDictBase):