Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add JsonValidationMiddleware to retry malformed JSON responses (#8215)

Adrian294cada6c7bb
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e23279f2a..f045dd9fe 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -848,6 +848,11 @@ Default: ``"latin-1"``
The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
+JsonValidationMiddleware
+------------------------
+
+.. autoclass:: scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware
+
OffsiteMiddleware
-----------------
@@ -1124,6 +1129,7 @@ Default::
'scrapy.exceptions.DownloadFailedError',
'scrapy.exceptions.DownloadTimeoutError',
'scrapy.exceptions.ResponseDataLossError',
+ 'json.JSONDecodeError',
'twisted.internet.error.ConnectionDone',
'twisted.internet.error.ConnectError',
'twisted.internet.error.ConnectionLost',
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 982f427ff..8117086f4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -933,6 +933,7 @@ Default:
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+ "scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware": 560,
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
diff --git a/scrapy/downloadermiddlewares/jsonvalidation.py b/scrapy/downloadermiddlewares/jsonvalidation.py
new file mode 100644
index 000000000..748a4625b
--- /dev/null
+++ b/scrapy/downloadermiddlewares/jsonvalidation.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from scrapy.exceptions import NotConfigured
+from scrapy.http import JsonResponse
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ from scrapy import Request
+ from scrapy.crawler import Crawler
+ from scrapy.http import Response
+
+
+class JsonValidationMiddleware:
+ """Raise :exc:`json.JSONDecodeError` for a successful
+ :class:`~scrapy.http.JsonResponse` whose body is not valid JSON.
+
+ .. versionadded:: VERSION
+
+ .. setting:: JSONVALIDATION_ENABLED
+
+ Set :setting:`JSONVALIDATION_ENABLED` to ``True`` to enable it, and
+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` to ``True`` for
+ :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` to retry those
+ responses:
+
+ .. code-block:: python
+
+ JSONVALIDATION_ENABLED = True
+ DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = True
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ if not crawler.settings.getbool("JSONVALIDATION_ENABLED"):
+ raise NotConfigured
+ return cls()
+
+ def process_response(self, request: Request, response: Response) -> Response:
+ if (
+ isinstance(response, JsonResponse)
+ and request.method != "HEAD"
+ and 200 <= response.status < 300
+ and response.status != 204
+ ):
+ response.json()
+ return response
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 19a51adcd..277c49e80 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -138,6 +138,7 @@ __all__ = [
"ITEM_PIPELINES_BASE",
"ITEM_PROCESSOR",
"JOBDIR",
+ "JSONVALIDATION_ENABLED",
"LOGSTATS_INTERVAL",
"LOG_COLOR",
"LOG_DATEFORMAT",
@@ -335,6 +336,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+ "scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware": 560,
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
@@ -455,6 +457,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
JOBDIR = None
+JSONVALIDATION_ENABLED = False
+
LOG_COLOR = True
LOG_ENABLED = True
LOG_INSTALL_ROOT_HANDLER = True
@@ -537,6 +541,7 @@ RETRY_EXCEPTIONS = [
"scrapy.exceptions.DownloadFailedError",
"scrapy.exceptions.DownloadTimeoutError",
"scrapy.exceptions.ResponseDataLossError",
+ "json.JSONDecodeError",
"twisted.internet.error.ConnectionDone",
"twisted.internet.error.ConnectError",
"twisted.internet.error.ConnectionLost",
diff --git a/tests/test_downloadermiddleware_jsonvalidation.py b/tests/test_downloadermiddleware_jsonvalidation.py
new file mode 100644
index 000000000..ce4609553
--- /dev/null
+++ b/tests/test_downloadermiddleware_jsonvalidation.py
@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+from gzip import compress
+
+import pytest
+
+from scrapy.downloadermiddlewares.jsonvalidation import JsonValidationMiddleware
+from scrapy.exceptions import NotConfigured
+from scrapy.http import JsonResponse, Request, Response, TextResponse
+from scrapy.spiders import Spider
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
+from tests.test_downloadermiddleware import TestManagerBase
+from tests.utils.decorators import coroutine_test
+
+URL = "https://example.com"
+
+
+def _mw() -> JsonValidationMiddleware:
+ crawler = get_crawler(Spider, {"JSONVALIDATION_ENABLED": True})
+ return build_from_crawler(JsonValidationMiddleware, crawler)
+
+
+def test_disabled_by_default() -> None:
+ with pytest.raises(NotConfigured):
+ build_from_crawler(JsonValidationMiddleware, get_crawler(Spider))
+
+
+def test_valid() -> None:
+ response = JsonResponse(URL, body=b'{"a": 1}')
+ assert _mw().process_response(Request(URL), response) is response
+
+
[email protected]("body", [b"", b"{", b"<html></html>"])
+def test_invalid(body: bytes) -> None:
+ response = JsonResponse(URL, body=body)
+ with pytest.raises(json.JSONDecodeError):
+ _mw().process_response(Request(URL), response)
+
+
+ ("method", "status", "cls"),
+ [
+ ("GET", 200, Response),
+ ("GET", 200, TextResponse),
+ ("GET", 204, JsonResponse),
+ ("GET", 404, JsonResponse),
+ ("GET", 503, JsonResponse),
+ ("HEAD", 200, JsonResponse),
+ ],
+)
+def test_skipped(method: str, status: int, cls: type[Response]) -> None:
+ response = cls(URL, status=status, body=b"{")
+ result = _mw().process_response(Request(URL, method=method), response)
+ assert result is response
+
+
+class TestChain(TestManagerBase):
+ settings_dict = {
+ "JSONVALIDATION_ENABLED": True,
+ "DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": True,
+ }
+
+ @coroutine_test
+ async def test_retry(self) -> None:
+ req = Request(URL)
+ async with self.get_mwman() as mwman:
+ result = await self._download(mwman, req, JsonResponse(URL, body=b"{"))
+ assert isinstance(result, Request)
+ assert result.meta["retry_times"] == 1
+
+ @coroutine_test
+ async def test_compressed(self) -> None:
+ req = Request(URL)
+ resp = Response(
+ URL,
+ body=compress(b'{"a": 1}'),
+ headers={"Content-Type": "application/json", "Content-Encoding": "gzip"},
+ )
+ async with self.get_mwman() as mwman:
+ result = await self._download(mwman, req, resp)
+ assert isinstance(result, JsonResponse)
+ assert result.json() == {"a": 1}