Scrapy, a fast high-level web crawling & scraping framework for Python.
Fall back to the response encoding in TextResponse.json() (#7897)
Adrian18ed0c0f7cea
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex f810146e4..a177d1ad5 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -1432,9 +1432,6 @@ TextResponse objects.. automethod:: TextResponse.json()- Returns a Python object from deserialized JSON document.- The result is cached after the first call.-.. method:: TextResponse.urljoin(url)Constructs an absolute url by combining the Response's base url withdiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex d01e23e47..64251780a 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -84,9 +84,21 @@ class TextResponse(Response):)def json(self) -> Any:- """Deserialize a JSON document to a Python object."""+ """Deserialize a JSON document to a Python object.++ .. versionchanged:: VERSION+ Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the+ JSON specification requires, are now decoded using+ :attr:`TextResponse.encoding` instead of raising+ :exc:`UnicodeDecodeError`.++ The result is cached after the first call.+ """if self._cached_decoded_json is _NONE:- self._cached_decoded_json = json.loads(self.body)+ try:+ self._cached_decoded_json = json.loads(self.body)+ except UnicodeDecodeError:+ self._cached_decoded_json = json.loads(self.text)return self._cached_decoded_json@propertydiff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex efa63e049..f705dbcee 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -481,6 +481,22 @@ class TestTextResponse(TestResponseBase):):text_response.json()+ def test_json_response_non_utf8(self):+ response = self.response_class(+ "http://www.example.com",+ body='{"message": "café"}'.encode("cp1252"),+ headers={"Content-Type": "application/json"},+ )+ assert response.json() == {"message": "café"}++ def test_json_response_wrong_charset(self):+ response = self.response_class(+ "http://www.example.com",+ body='{"message": "café"}'.encode(),+ headers={"Content-Type": "application/json; charset=iso-8859-1"},+ )+ assert response.json() == {"message": "café"}+def test_cache_json_response(self):json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]for json_body in json_valid_bodies: