Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fall back to the response encoding in TextResponse.json() (#7897)

Adrian18ed0c0f7cea
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f810146e4..a177d1ad5 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1432,9 +1432,6 @@ TextResponse objects
.. automethod:: TextResponse.json()
- Returns a Python object from deserialized JSON document.
- The result is cached after the first call.
-
.. method:: TextResponse.urljoin(url)
Constructs an absolute url by combining the Response's base url with
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index d01e23e47..64251780a 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -84,9 +84,21 @@ class TextResponse(Response):
)
def json(self) -> Any:
- """Deserialize a JSON document to a Python object."""
+ """Deserialize a JSON document to a Python object.
+
+ .. versionchanged:: VERSION
+ Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the
+ JSON specification requires, are now decoded using
+ :attr:`TextResponse.encoding` instead of raising
+ :exc:`UnicodeDecodeError`.
+
+ The result is cached after the first call.
+ """
if self._cached_decoded_json is _NONE:
- self._cached_decoded_json = json.loads(self.body)
+ try:
+ self._cached_decoded_json = json.loads(self.body)
+ except UnicodeDecodeError:
+ self._cached_decoded_json = json.loads(self.text)
return self._cached_decoded_json
@property
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index efa63e049..f705dbcee 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -481,6 +481,22 @@ class TestTextResponse(TestResponseBase):
):
text_response.json()
+ def test_json_response_non_utf8(self):
+ response = self.response_class(
+ "http://www.example.com",
+ body='{"message": "café"}'.encode("cp1252"),
+ headers={"Content-Type": "application/json"},
+ )
+ assert response.json() == {"message": "café"}
+
+ def test_json_response_wrong_charset(self):
+ response = self.response_class(
+ "http://www.example.com",
+ body='{"message": "café"}'.encode(),
+ headers={"Content-Type": "application/json; charset=iso-8859-1"},
+ )
+ assert response.json() == {"message": "café"}
+
def test_cache_json_response(self):
json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
for json_body in json_valid_bodies: