Scrapy, a fast high-level web crawling & scraping framework for Python.
Let process_response() exceptions reach process_exception() (#8177)
Adrian7af73d44bf6b
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 67b606d84..e23279f2a 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -118,9 +118,15 @@ defines one or more of these methods:halted and the returned request is rescheduled to be downloaded in the future.This is the same behavior as if a request is returned from :meth:`process_request`.- If it raises an :exc:`~scrapy.exceptions.IgnoreRequest` exception, the errback- function of the request (``Request.errback``) is called. If no code handles the raised- exception, it is ignored and not logged (unlike other exceptions).+ If it raises an exception, and the+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` setting is ``True``,+ the :meth:`process_exception` methods of the downloader middlewares that+ have not processed the response yet are called.++ If no middleware handles the exception, the errback function of the+ request (``Request.errback``) is called. If no code handles an+ :exc:`~scrapy.exceptions.IgnoreRequest` exception, it is ignored and not+ logged, unlike other exceptions.:param request: the request that originated the response:type request: is a :class:`~scrapy.Request` object@@ -133,7 +139,9 @@ defines one or more of these methods:Scrapy calls :meth:`process_exception` when a :ref:`download handler<topics-download-handlers>` or a :meth:`process_request` (from adownloader middleware) raises an exception (including an- :exc:`~scrapy.exceptions.IgnoreRequest` exception).+ :exc:`~scrapy.exceptions.IgnoreRequest` exception), and, if the+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` setting is ``True``,+ when a :meth:`process_response` does.:meth:`process_exception` should return: either ``None``,a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.@@ -142,9 +150,12 @@ defines one or more of these methods:executing any other :meth:`process_exception` methods of installed middleware,until no middleware is left and the default exception handling kicks in.- If it returns a :class:`~scrapy.http.Response` object, the :meth:`process_response`- method chain of installed middleware is started, and Scrapy won't bother calling- any other :meth:`process_exception` methods of middleware.+ If it returns a :class:`~scrapy.http.Response` object, the+ :meth:`process_response` method chain of installed middleware is started,+ and Scrapy won't bother calling any other :meth:`process_exception`+ methods of middleware. For an exception from a :meth:`process_response`,+ that chain resumes at the middlewares that have not processed the+ response yet.If it returns a :class:`~scrapy.Request` object, the returned request isrescheduled to be downloaded in the future. This stops the execution of@@ -1108,6 +1119,7 @@ Default::['scrapy.exceptions.CannotResolveHostError',+ 'scrapy.exceptions.DecompressionError','scrapy.exceptions.DownloadConnectionRefusedError','scrapy.exceptions.DownloadFailedError','scrapy.exceptions.DownloadTimeoutError',diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rstindex 0aab90a43..c7364539f 100644--- a/docs/topics/exceptions.rst+++ b/docs/topics/exceptions.rst@@ -12,6 +12,8 @@ the :ref:`download handler exceptions <download-handlers-exceptions>`... autoexception:: CloseSpider+.. autoexception:: DecompressionError+.. autoexception:: DontCloseSpider.. autoexception:: DropItemdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex dbc60c952..982f427ff 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -948,6 +948,32 @@ should never modify this setting in your project, modify:setting:`DOWNLOADER_MIDDLEWARES` instead. For more info see:ref:`topics-downloader-middleware-setting`.+.. setting:: DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS++DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS+-----------------------------------------++.. versionadded:: VERSION++Default: ``False``++Whether an exception raised by the+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`+method of a downloader middleware is passed to the+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`+method of the downloader middlewares that have not processed the response yet.++Enabling this lets :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`+retry those exceptions, e.g. a response that cannot be decompressed.++Before enabling it, check that the ``process_exception`` methods of your+downloader middlewares handle those exceptions as intended. They also get the+:exc:`~scrapy.exceptions.IgnoreRequest` exceptions that middlewares raise to+drop a response, so one that returns a request for every exception it gets+turns such a drop into a new request.++``True`` will become the only supported value in a future version of Scrapy.+.. setting:: DOWNLOADER_STATSDOWNLOADER_STATSdiff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex a71589d4a..3b9f4e11e 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -8,9 +8,10 @@ from __future__ import annotationsimport warningsfrom functools import wraps+from itertools import islicefrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list@@ -28,12 +29,21 @@ if TYPE_CHECKING:from twisted.internet.defer import Deferredfrom scrapy import Spider+ from scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettingsclass DownloaderMiddlewareManager(MiddlewareManager):component_name = "downloader middleware"+ def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:+ super().__init__(*middlewares, crawler=crawler)+ self._response_exceptions = (+ crawler.settings.getbool("DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS")+ if crawler is not None+ else False+ )+@classmethoddef _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:return build_component_list(@@ -44,12 +54,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):if hasattr(mw, "process_request"):self.methods["process_request"].append(mw.process_request)self._check_mw_method_spider_arg(mw.process_request)- if hasattr(mw, "process_response"):- self.methods["process_response"].appendleft(mw.process_response)- self._check_mw_method_spider_arg(mw.process_response)- if hasattr(mw, "process_exception"):- self.methods["process_exception"].appendleft(mw.process_exception)- self._check_mw_method_spider_arg(mw.process_exception)+ process_response = getattr(mw, "process_response", None)+ self.methods["process_response"].appendleft(process_response)+ if process_response is not None:+ self._check_mw_method_spider_arg(process_response)+ process_exception = getattr(mw, "process_exception", None)+ self.methods["process_exception"].appendleft(process_exception)+ if process_exception is not None:+ self._check_mw_method_spider_arg(process_exception)def download(self,@@ -84,7 +96,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):await _process_pending_io()# either returns a request or response (which we pass to process_response())# or reraises the exception- result = await self._process_exception(ex, request)+ result, _ = await self._process_exception(ex, request)return await self._process_response(result, request)def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:@@ -114,34 +126,96 @@ class DownloaderMiddlewareManager(MiddlewareManager):return await download_func(request)async def _process_response(- self, response: Response | Request, request: Request+ self, response: Response | Request, request: Request, index: int = 0) -> Response | Request:if response is None:raise TypeError("Received None in process_response")- if isinstance(response, Request):- return response-- for method in self.methods["process_response"]:- assert method is not None- response = await ensure_awaitable(- self._handle_mw_method(method, request=request, response=response),- _warn=global_object_name(method),- )- if not isinstance(response, (Response, Request)):+ methods = self.methods["process_response"]+ while index < len(methods):+ if isinstance(response, Request):+ return response+ method = methods[index]+ index += 1+ if method is None:+ continue+ try:+ result = await ensure_awaitable(+ self._handle_mw_method(method, request=request, response=response),+ _warn=global_object_name(method),+ )+ except Exception as ex:+ if not self._response_exceptions:+ # CloseSpider closes the spider either way, it never+ # reaches process_exception().+ if not isinstance(ex, CloseSpider):+ self._warn_response_exception(method, ex, index)+ raise+ response, index = await self._process_exception(ex, request, index)+ continue++ if not isinstance(result, (Response, Request)):raise _InvalidOutput(f"Middleware {method.__qualname__} must return Response or Request, "- f"got {type(response)}"+ f"got {type(result)}")- if isinstance(response, Request):- return response+ response = resultreturn response+ def _warn_response_exception(+ self, method: Callable[..., Any], exception: Exception, index: int+ ) -> None:+ """Warn that *exception*, raised by the process_response() *method* of a+ downloader middleware, will go to the process_exception() methods from+ *index* on once DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS is mandatory.++ Warn about nothing when all those methods are Scrapy's own: they are+ nothing for users to review, and RetryMiddleware, the one that acts on+ these exceptions, still lets them reach the request errback once+ retries are exhausted.+ """+ targets = [+ global_object_name(target)+ for target in islice(self.methods["process_exception"], index, None)+ if target is not None and not target.__module__.startswith("scrapy.")+ ]+ if not targets:+ return+ warnings.warn(+ f"{global_object_name(method)} raised"+ f" {global_object_name(type(exception))}. In a future Scrapy version,"+ " exceptions raised by the process_response() method of a downloader"+ " middleware will be passed to the process_exception() method of the"+ " downloader middlewares that have not processed the response yet,"+ " instead of reaching the request errback directly. Before setting the"+ " DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS setting to True to get"+ " that behavior now, make sure that this exception is handled as"+ " intended by the following methods, e.g. that an IgnoreRequest"+ " exception raised to drop a response is not retried:"+ f" {', '.join(targets)}.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+async def _process_exception(- self, exception: Exception, request: Request | Response- ) -> Response | Request:- for method in self.methods["process_exception"]:- assert method is not None+ self, exception: Exception, request: Request, index: int = 0+ ) -> tuple[Response | Request, int]:+ """Let middlewares handle *exception*, starting at *index*.++ Return the result of the middleware that handled it and the index of+ the next middleware, or reraise *exception* if none handled it.+ """+ # CloseSpider asks the engine to close the spider, it is not a download+ # error for middlewares to recover from.+ if isinstance(exception, CloseSpider):+ raise exception++ methods = self.methods["process_exception"]+ while index < len(methods):+ method = methods[index]+ index += 1+ if method is None:+ continueresponse = await ensure_awaitable(self._handle_mw_method(method, request=request, exception=exception),_warn=global_object_name(method),@@ -152,5 +226,5 @@ class DownloaderMiddlewareManager(MiddlewareManager):f"Request, got {type(response)}")if response:- return response+ return response, indexraise exceptiondiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex bfdda63ca..6c68fc65a 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -263,6 +263,14 @@ class Scraper:await self.handle_spider_output_async(output, request, result)return+ if result.check(CloseSpider):+ exc = result.value+ assert isinstance(exc, CloseSpider) # typing+ _schedule_coro(+ self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")+ )+ return+try:# call the request errback with the downloader erroroutput = await self.call_spider_async(result, request)diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 674384a26..f01c8a1f9 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -6,10 +6,16 @@ from logging import getLoggerfrom typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signals-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import (+ DecompressionError,+ IgnoreRequest,+ NotConfigured,+ ScrapyDeprecationWarning,+)from scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._compression import (+ _DECOMPRESSION_ERRORS,_DecompressionMaxSizeExceeded,_inflate,_unbrotli,@@ -104,6 +110,11 @@ class HttpCompressionMiddleware:)logger.warning(msg)raise IgnoreRequest(msg) from e+ except _DECOMPRESSION_ERRORS as e:+ encodings = b", ".join(content_encoding).decode()+ raise DecompressionError(+ f"Could not decompress {response} ({encodings}): {e}"+ ) from eif len(response.body) < warn_size <= len(decoded_body):logger.warning(f"{response} body size after decompression "diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 70d284188..db8b5d196 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -56,12 +56,16 @@ class DontCloseSpider(Exception):class CloseSpider(Exception):- """Raised from a :ref:`spider callback <topics-spiders>`, or while the+ """Raised from a :ref:`spider callback <topics-spiders>`, a+ :ref:`downloader middleware <topics-downloader-middleware>`, or while thespider is starting, to request the spider to be closed/stopped... versionchanged:: 2.18.0Added support for raising it while the spider is starting.+ .. versionchanged:: VERSION+ Added support for raising it from a downloader middleware.+*reason* is a string with the reason for closing.For example:@@ -133,6 +137,15 @@ class ResponseDataLossError(Exception):"""Indicates that Scrapy couldn't get a complete response."""+class DecompressionError(Exception):+ """Raised by+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`+ when a response body cannot be decompressed.++ .. versionadded:: VERSION+ """++class UnsupportedURLSchemeError(Exception):"""Indicates that the URL scheme is not supported."""diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 0c0216ed2..19a51adcd 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -65,6 +65,7 @@ __all__ = ["DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING","DOWNLOADER_MIDDLEWARES","DOWNLOADER_MIDDLEWARES_BASE",+ "DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS","DOWNLOADER_STATS","DOWNLOAD_BIND_ADDRESS","DOWNLOAD_DELAY",@@ -343,6 +344,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {"scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,# Downloader side}+DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = FalseDOWNLOADER_STATS = True@@ -530,6 +532,7 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"RETRY_ENABLED = TrueRETRY_EXCEPTIONS = ["scrapy.exceptions.CannotResolveHostError",+ "scrapy.exceptions.DecompressionError","scrapy.exceptions.DownloadConnectionRefusedError","scrapy.exceptions.DownloadFailedError","scrapy.exceptions.DownloadTimeoutError",@@ -537,8 +540,6 @@ RETRY_EXCEPTIONS = ["twisted.internet.error.ConnectionDone","twisted.internet.error.ConnectError","twisted.internet.error.ConnectionLost",- # OSError is raised by the HttpCompression middleware when trying to- # decompress an empty responseOSError,"scrapy.core.downloader.handlers.http11.TunnelError",]diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex 0432a7231..be874b3df 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -84,4 +84,5 @@ DOWNLOAD_DELAY = 1#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"# Set settings whose default value is deprecated to a future-proof value+DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = TrueFEED_EXPORT_ENCODING = "utf-8"diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 1966b4f1c..2be0bd434 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -23,6 +23,15 @@ if TYPE_CHECKING:_CHUNK_SIZE = 65536 # 64 KiB+_DECOMPRESSION_ERRORS: tuple[type[Exception], ...] = (+ EOFError,+ OSError,+ brotli.error,+ struct.error,+ zlib.error,+ zstd.ZstdError,+)+class _DecompressionMaxSizeExceeded(ValueError):def __init__(self, decompressed_size: int, max_size: int) -> None:diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 018b82f23..da0a08b7b 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -67,6 +67,16 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer+class CloseSpiderProcessRequestMiddleware:+ def process_request(self, request):+ raise CloseSpider("my_reason")+++class CloseSpiderProcessResponseMiddleware:+ def process_response(self, request, response):+ raise CloseSpider("my_reason")++class TestCrawl:@coroutine_testasync def test_follow_all(self, mockserver: MockServer) -> None:@@ -1025,3 +1035,22 @@ class TestCrawlSpider:await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb)assert "Closing spider (my_reason)" in caplog.textassert "Spider error processing" not in caplog.text++ @pytest.mark.parametrize(+ "middleware",+ [CloseSpiderProcessRequestMiddleware, CloseSpiderProcessResponseMiddleware],+ )+ @coroutine_test+ async def test_raise_closespider_downloader_middleware(+ self,+ caplog: pytest.LogCaptureFixture,+ mockserver: MockServer,+ middleware: type,+ ) -> None:+ crawler = get_crawler(+ SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {middleware: 1}}+ )+ with caplog.at_level(logging.INFO):+ await crawler.crawl_async(seed=mockserver.url("/"))+ assert "Closing spider (my_reason)" in caplog.text+ assert "Error downloading" not in caplog.textdiff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 46cb6b920..163aa70d4 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -2,15 +2,21 @@ from __future__ import annotationsimport asynciofrom contextlib import asynccontextmanager-from gzip import BadGzipFile-from typing import TYPE_CHECKING+from gzip import compress+from typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytestfrom twisted.internet.defer import Deferred, succeedfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManager-from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput+from scrapy.exceptions import (+ CloseSpider,+ DecompressionError,+ IgnoreRequest,+ ScrapyDeprecationWarning,+ _InvalidOutput,+)from scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.defer import maybe_deferred_to_future@@ -23,8 +29,37 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator+def _invalid_gzipped_response(request: Request) -> Response:+ body = b"<p>You are being redirected</p>"+ return Response(+ request.url,+ status=200,+ body=body,+ headers={+ "Content-Length": str(len(body)),+ "Content-Type": "text/html",+ "Content-Encoding": "gzip",+ "Location": "http://example.com/login",+ },+ )+++def _truncated_gzipped_response(request: Request) -> Response:+ body = compress(b"<p>You are being redirected</p>")[:10]+ return Response(+ request.url,+ status=200,+ body=body,+ headers={+ "Content-Length": str(len(body)),+ "Content-Type": "text/html",+ "Content-Encoding": "gzip",+ },+ )++class TestManagerBase:- settings_dict = None+ settings_dict: dict[str, Any] | None = None# should be a fixture but async fixtures that use Futures are problematic with pytest-twisted@asynccontextmanager@@ -117,19 +152,8 @@ class TestDefaults(TestManagerBase):@coroutine_testasync def test_200_and_invalid_gzipped_body_must_fail(self):req = Request("http://example.com")- body = b"<p>You are being redirected</p>"- resp = Response(- req.url,- status=200,- body=body,- headers={- "Content-Length": str(len(body)),- "Content-Type": "text/html",- "Content-Encoding": "gzip",- "Location": "http://example.com/login",- },- )- with pytest.raises(BadGzipFile):+ resp = _invalid_gzipped_response(req)+ with pytest.raises(DecompressionError):async with self.get_mwman() as mwman:await self._download(mwman, req, resp)@@ -185,6 +209,219 @@ class TestResponseFromProcessException(TestManagerBase):]+class TestResponseExceptions(TestManagerBase):+ """Tests exceptions from process_response reaching process_exception."""++ settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": True}++ @coroutine_test+ async def test_request_from_process_exception(self):+ req = Request("http://example.com/index.html")+ retry_req = Request("http://example.com/index.html")+ calls = []++ class OuterMiddleware:+ def process_response(self, request, response):+ calls.append("outer.process_response")+ return response++ def process_exception(self, request, exception):+ calls.append("outer.process_exception")+ return retry_req++ class InnerMiddleware:+ def process_response(self, request, response):+ calls.append("inner.process_response")+ raise ValueError("test")++ def process_exception(self, request, exception):+ calls.append("inner.process_exception")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(OuterMiddleware())+ mwman._add_middleware(InnerMiddleware())+ result = await self._download(mwman, req)+ assert result is retry_req+ assert calls == ["inner.process_response", "outer.process_exception"]++ @coroutine_test+ async def test_response_from_process_exception(self):+ req = Request("http://example.com/index.html")+ recovered = Response("http://example.com/index.html")+ calls = []++ class OuterMiddleware:+ def process_response(self, request, response):+ calls.append("outer.process_response")+ return response++ class MiddleMiddleware:+ def process_response(self, request, response):+ calls.append("middle.process_response")+ return response++ def process_exception(self, request, exception):+ calls.append("middle.process_exception")+ return recovered++ class InnerMiddleware:+ def process_response(self, request, response):+ calls.append("inner.process_response")+ raise ValueError("test")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(OuterMiddleware())+ mwman._add_middleware(MiddleMiddleware())+ mwman._add_middleware(InnerMiddleware())+ result = await self._download(mwman, req)+ assert result is recovered+ assert calls == [+ "inner.process_response",+ "middle.process_exception",+ "outer.process_response",+ ]++ @coroutine_test+ async def test_processed_middleware_not_called(self):+ req = Request("http://example.com/index.html")+ calls = []++ class RaisingMiddleware:+ def process_response(self, request, response):+ calls.append("raising.process_response")+ raise ValueError("test")++ class InnerMiddleware:+ def process_response(self, request, response):+ calls.append("inner.process_response")+ return response++ def process_exception(self, request, exception):+ calls.append("inner.process_exception")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(RaisingMiddleware())+ mwman._add_middleware(InnerMiddleware())+ with pytest.raises(ValueError, match="test"):+ await self._download(mwman, req)+ assert calls == ["inner.process_response", "raising.process_response"]++ @coroutine_test+ async def test_unhandled(self):+ req = Request("http://example.com/index.html")++ class RaisingMiddleware:+ def process_response(self, request, response):+ raise ValueError("test")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(RaisingMiddleware())+ with pytest.raises(ValueError, match="test"):+ await self._download(mwman, req)++ @pytest.mark.parametrize(+ "response_func", [_invalid_gzipped_response, _truncated_gzipped_response]+ )+ @coroutine_test+ async def test_retry(self, response_func):+ """Decompression failures are retried, as RETRY_EXCEPTIONS promises."""+ req = Request("http://example.com")+ async with self.get_mwman() as mwman:+ result = await self._download(mwman, req, response_func(req))+ assert isinstance(result, Request)+ assert result.url == req.url++ @coroutine_test+ async def test_close_spider(self):+ req = Request("http://example.com/index.html")+ calls = []++ class OuterMiddleware:+ def process_exception(self, request, exception):+ calls.append("outer.process_exception")++ class InnerMiddleware:+ def process_response(self, request, response):+ raise CloseSpider("test")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(OuterMiddleware())+ mwman._add_middleware(InnerMiddleware())+ with pytest.raises(CloseSpider):+ await self._download(mwman, req)+ assert not calls++ @coroutine_test+ async def test_close_spider_from_process_request(self):+ req = Request("http://example.com/index.html")+ calls = []++ class Middleware:+ def process_request(self, request):+ raise CloseSpider("test")++ def process_exception(self, request, exception):+ calls.append("process_exception")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(Middleware())+ with pytest.raises(CloseSpider):+ await self._download(mwman, req)+ assert not calls+++class TestResponseExceptionsDisabled(TestManagerBase):+ """Tests the deprecated behavior, which the default value still gives."""++ @pytest.mark.parametrize("exception", [ValueError, IgnoreRequest])+ @coroutine_test+ async def test_not_passed_to_process_exception(self, exception):+ req = Request("http://example.com/index.html")+ calls = []++ class OuterMiddleware:+ def process_exception(self, request, exception):+ calls.append("outer.process_exception")++ class InnerMiddleware:+ def process_response(self, request, response):+ raise exception("test")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(OuterMiddleware())+ mwman._add_middleware(InnerMiddleware())+ with (+ pytest.raises(exception, match="test"),+ pytest.warns(+ ScrapyDeprecationWarning,+ match="OuterMiddleware.process_exception",+ ),+ ):+ await self._download(mwman, req)+ assert not calls++ @coroutine_test+ async def test_only_built_in_process_exception(self):+ """Nothing is warned about when only Scrapy's own process_exception()+ methods would get the exception."""+ req = Request("http://example.com/index.html")++ class RaisingMiddleware:+ def process_response(self, request, response):+ raise ValueError("test")++ async with self.get_mwman() as mwman:+ mwman._add_middleware(RaisingMiddleware())+ with pytest.raises(ValueError, match="test"):+ await self._download(mwman, req)+++class TestResponseExceptionsDisabledExplicitly(TestResponseExceptionsDisabled):+ """Asking for the deprecated behavior does not silence the warning."""++ settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": False}++class TestInvalidOutput(TestManagerBase):@coroutine_testasync def test_invalid_process_request(self):diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 285982c46..96194ee86 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -13,7 +13,12 @@ from scrapy.downloadermiddlewares.httpcompression import (ACCEPTED_ENCODINGS,HttpCompressionMiddleware,)-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import (+ DecompressionError,+ IgnoreRequest,+ NotConfigured,+ ScrapyDeprecationWarning,+)from scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider@@ -809,3 +814,22 @@ class TestHttpCompression:continueresp = self._get_truncated_response(check_key)assert len(resp.body) == 0++ @pytest.mark.parametrize(+ ("body", "encoding"),+ [+ (b"not compressed", "gzip"),+ (b"\x1f\x8b\x08\x00", "gzip"),+ (b"\x1f\x8b\x08" + b"\x00" * 7 + b"\xff" * 20, "gzip"),+ (b"not compressed", "deflate"),+ (b"not compressed", "br"),+ (b"not compressed", "zstd"),+ ],+ )+ def test_process_response_invalid(self, body, encoding):+ request = Request("http://example.com")+ response = Response(+ "http://example.com", body=body, headers={"Content-Encoding": encoding}+ )+ with pytest.raises(DecompressionError):+ self.mw.process_response(request, response)