Scrapy, a fast high-level web crawling & scraping framework for Python.
Add JsonValidationMiddleware to retry malformed JSON responses (#8215)
Adrian294cada6c7bb
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex e23279f2a..f045dd9fe 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -848,6 +848,11 @@ Default: ``"latin-1"``The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.+JsonValidationMiddleware+------------------------++.. autoclass:: scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware+OffsiteMiddleware-----------------@@ -1124,6 +1129,7 @@ Default::'scrapy.exceptions.DownloadFailedError','scrapy.exceptions.DownloadTimeoutError','scrapy.exceptions.ResponseDataLossError',+ 'json.JSONDecodeError','twisted.internet.error.ConnectionDone','twisted.internet.error.ConnectError','twisted.internet.error.ConnectionLost',diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 982f427ff..8117086f4 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -933,6 +933,7 @@ Default:"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,+ "scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware": 560,"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,diff --git a/scrapy/downloadermiddlewares/jsonvalidation.py b/scrapy/downloadermiddlewares/jsonvalidation.pynew file mode 100644index 000000000..748a4625b--- /dev/null+++ b/scrapy/downloadermiddlewares/jsonvalidation.py@@ -0,0 +1,50 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from scrapy.exceptions import NotConfigured+from scrapy.http import JsonResponse++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy import Request+ from scrapy.crawler import Crawler+ from scrapy.http import Response+++class JsonValidationMiddleware:+ """Raise :exc:`json.JSONDecodeError` for a successful+ :class:`~scrapy.http.JsonResponse` whose body is not valid JSON.++ .. versionadded:: VERSION++ .. setting:: JSONVALIDATION_ENABLED++ Set :setting:`JSONVALIDATION_ENABLED` to ``True`` to enable it, and+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` to ``True`` for+ :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` to retry those+ responses:++ .. code-block:: python++ JSONVALIDATION_ENABLED = True+ DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = True+ """++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ if not crawler.settings.getbool("JSONVALIDATION_ENABLED"):+ raise NotConfigured+ return cls()++ def process_response(self, request: Request, response: Response) -> Response:+ if (+ isinstance(response, JsonResponse)+ and request.method != "HEAD"+ and 200 <= response.status < 300+ and response.status != 204+ ):+ response.json()+ return responsediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 19a51adcd..277c49e80 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -138,6 +138,7 @@ __all__ = ["ITEM_PIPELINES_BASE","ITEM_PROCESSOR","JOBDIR",+ "JSONVALIDATION_ENABLED","LOGSTATS_INTERVAL","LOG_COLOR","LOG_DATEFORMAT",@@ -335,6 +336,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,+ "scrapy.downloadermiddlewares.jsonvalidation.JsonValidationMiddleware": 560,"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,@@ -455,6 +457,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"JOBDIR = None+JSONVALIDATION_ENABLED = False+LOG_COLOR = TrueLOG_ENABLED = TrueLOG_INSTALL_ROOT_HANDLER = True@@ -537,6 +541,7 @@ RETRY_EXCEPTIONS = ["scrapy.exceptions.DownloadFailedError","scrapy.exceptions.DownloadTimeoutError","scrapy.exceptions.ResponseDataLossError",+ "json.JSONDecodeError","twisted.internet.error.ConnectionDone","twisted.internet.error.ConnectError","twisted.internet.error.ConnectionLost",diff --git a/tests/test_downloadermiddleware_jsonvalidation.py b/tests/test_downloadermiddleware_jsonvalidation.pynew file mode 100644index 000000000..ce4609553--- /dev/null+++ b/tests/test_downloadermiddleware_jsonvalidation.py@@ -0,0 +1,84 @@+from __future__ import annotations++import json+from gzip import compress++import pytest++from scrapy.downloadermiddlewares.jsonvalidation import JsonValidationMiddleware+from scrapy.exceptions import NotConfigured+from scrapy.http import JsonResponse, Request, Response, TextResponse+from scrapy.spiders import Spider+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawler+from tests.test_downloadermiddleware import TestManagerBase+from tests.utils.decorators import coroutine_test++URL = "https://example.com"+++def _mw() -> JsonValidationMiddleware:+ crawler = get_crawler(Spider, {"JSONVALIDATION_ENABLED": True})+ return build_from_crawler(JsonValidationMiddleware, crawler)+++def test_disabled_by_default() -> None:+ with pytest.raises(NotConfigured):+ build_from_crawler(JsonValidationMiddleware, get_crawler(Spider))+++def test_valid() -> None:+ response = JsonResponse(URL, body=b'{"a": 1}')+ assert _mw().process_response(Request(URL), response) is response++[email protected]("body", [b"", b"{", b"<html></html>"])+def test_invalid(body: bytes) -> None:+ response = JsonResponse(URL, body=body)+ with pytest.raises(json.JSONDecodeError):+ _mw().process_response(Request(URL), response)+++ ("method", "status", "cls"),+ [+ ("GET", 200, Response),+ ("GET", 200, TextResponse),+ ("GET", 204, JsonResponse),+ ("GET", 404, JsonResponse),+ ("GET", 503, JsonResponse),+ ("HEAD", 200, JsonResponse),+ ],+)+def test_skipped(method: str, status: int, cls: type[Response]) -> None:+ response = cls(URL, status=status, body=b"{")+ result = _mw().process_response(Request(URL, method=method), response)+ assert result is response+++class TestChain(TestManagerBase):+ settings_dict = {+ "JSONVALIDATION_ENABLED": True,+ "DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": True,+ }++ @coroutine_test+ async def test_retry(self) -> None:+ req = Request(URL)+ async with self.get_mwman() as mwman:+ result = await self._download(mwman, req, JsonResponse(URL, body=b"{"))+ assert isinstance(result, Request)+ assert result.meta["retry_times"] == 1++ @coroutine_test+ async def test_compressed(self) -> None:+ req = Request(URL)+ resp = Response(+ URL,+ body=compress(b'{"a": 1}'),+ headers={"Content-Type": "application/json", "Content-Encoding": "gzip"},+ )+ async with self.get_mwman() as mwman:+ result = await self._download(mwman, req, resp)+ assert isinstance(result, JsonResponse)+ assert result.json() == {"a": 1}