Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix CaseInsensitiveDict.copy() sharing state with the original (#7783)
Shadow_Lu59ebb26e60ea
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex c020ff4b9..e761a2474 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -132,6 +132,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):def __repr__(self) -> str:return f"<{self.__class__.__name__}: {super().__repr__()}>"+ # UserDict.copy() shallow-copies the instance, which would share self._keys+ # between the copy and the original.+ def __copy__(self) -> Self:+ new = self.__class__()+ new.data = self.data.copy()+ new._keys = self._keys.copy()+ return new++ copy = __copy__++ # UserDict.__ior__ updates self.data directly, which would leave self._keys+ # out of date.+ def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]+ self.update(other)+ return self+def _normkey(self, key: str | bytes) -> str | bytes:return keydiff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex fe1f60c7d..f43d20e69 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -204,6 +204,15 @@ class TestCaseInsensitiveDictBase(ABC):assert h1.get("header1") == h3.get("header1")assert h1.get("header1") == h3.get("HEADER1")+ def test_copy_is_independent(self):+ h1 = self.dict_class({"header1": "value1", "header2": "value2"})+ for h2 in (copy.copy(h1), h1.copy()):+ del h2["header1"]+ h2["header3"] = "value3"+ assert "header1" in h1+ assert "header3" not in h1+ assert dict(h1) == {"header1": "value1", "header2": "value2"}+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):dict_class = CaseInsensitiveDict # type: ignore[assignment]@@ -220,6 +229,28 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):assert isinstance(iterkeys, Iterator)assert list(iterkeys) == ["AsDf", "FoO"]+ def test_copy_keeps_values(self):+ class MyDict(self.dict_class):+ def _normvalue(self, value):+ return value + 1++ d = MyDict({"key": 1})+ for copied in (copy.copy(d), d.copy()):+ assert copied["key"] == 2++ def test_ior(self):+ d = self.dict_class({"header1": "value1"})+ d |= {"HEADER1": "value2", "header2": "value3"}+ assert len(d) == 2+ assert d["HeAdEr1"] == "value2"+ assert d["HeAdEr2"] == "value3"++ def test_ior_mapping(self):+ d = self.dict_class({"header1": "value1"})+ d |= self.dict_class({"HEADER1": "value2"})+ assert len(d) == 1+ assert d["HeAdEr1"] == "value2"+@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")class TestCaselessDict(TestCaseInsensitiveDictBase):