Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let process_response() exceptions reach process_exception() (#8177)

Adrian7af73d44bf6b
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 67b606d84..e23279f2a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -118,9 +118,15 @@ defines one or more of these methods:
halted and the returned request is rescheduled to be downloaded in the future.
This is the same behavior as if a request is returned from :meth:`process_request`.
- If it raises an :exc:`~scrapy.exceptions.IgnoreRequest` exception, the errback
- function of the request (``Request.errback``) is called. If no code handles the raised
- exception, it is ignored and not logged (unlike other exceptions).
+ If it raises an exception, and the
+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` setting is ``True``,
+ the :meth:`process_exception` methods of the downloader middlewares that
+ have not processed the response yet are called.
+
+ If no middleware handles the exception, the errback function of the
+ request (``Request.errback``) is called. If no code handles an
+ :exc:`~scrapy.exceptions.IgnoreRequest` exception, it is ignored and not
+ logged, unlike other exceptions.
:param request: the request that originated the response
:type request: is a :class:`~scrapy.Request` object
@@ -133,7 +139,9 @@ defines one or more of these methods:
Scrapy calls :meth:`process_exception` when a :ref:`download handler
<topics-download-handlers>` or a :meth:`process_request` (from a
downloader middleware) raises an exception (including an
- :exc:`~scrapy.exceptions.IgnoreRequest` exception).
+ :exc:`~scrapy.exceptions.IgnoreRequest` exception), and, if the
+ :setting:`DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS` setting is ``True``,
+ when a :meth:`process_response` does.
:meth:`process_exception` should return: either ``None``,
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
@@ -142,9 +150,12 @@ defines one or more of these methods:
executing any other :meth:`process_exception` methods of installed middleware,
until no middleware is left and the default exception handling kicks in.
- If it returns a :class:`~scrapy.http.Response` object, the :meth:`process_response`
- method chain of installed middleware is started, and Scrapy won't bother calling
- any other :meth:`process_exception` methods of middleware.
+ If it returns a :class:`~scrapy.http.Response` object, the
+ :meth:`process_response` method chain of installed middleware is started,
+ and Scrapy won't bother calling any other :meth:`process_exception`
+ methods of middleware. For an exception from a :meth:`process_response`,
+ that chain resumes at the middlewares that have not processed the
+ response yet.
If it returns a :class:`~scrapy.Request` object, the returned request is
rescheduled to be downloaded in the future. This stops the execution of
@@ -1108,6 +1119,7 @@ Default::
[
'scrapy.exceptions.CannotResolveHostError',
+ 'scrapy.exceptions.DecompressionError',
'scrapy.exceptions.DownloadConnectionRefusedError',
'scrapy.exceptions.DownloadFailedError',
'scrapy.exceptions.DownloadTimeoutError',
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 0aab90a43..c7364539f 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -12,6 +12,8 @@ the :ref:`download handler exceptions <download-handlers-exceptions>`.
.. autoexception:: CloseSpider
+.. autoexception:: DecompressionError
+
.. autoexception:: DontCloseSpider
.. autoexception:: DropItem
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index dbc60c952..982f427ff 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -948,6 +948,32 @@ should never modify this setting in your project, modify
:setting:`DOWNLOADER_MIDDLEWARES` instead. For more info see
:ref:`topics-downloader-middleware-setting`.
+.. setting:: DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS
+
+DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS
+-----------------------------------------
+
+.. versionadded:: VERSION
+
+Default: ``False``
+
+Whether an exception raised by the
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
+method of a downloader middleware is passed to the
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
+method of the downloader middlewares that have not processed the response yet.
+
+Enabling this lets :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`
+retry those exceptions, e.g. a response that cannot be decompressed.
+
+Before enabling it, check that the ``process_exception`` methods of your
+downloader middlewares handle those exceptions as intended. They also get the
+:exc:`~scrapy.exceptions.IgnoreRequest` exceptions that middlewares raise to
+drop a response, so one that returns a request for every exception it gets
+turns such a drop into a new request.
+
+``True`` will become the only supported value in a future version of Scrapy.
+
.. setting:: DOWNLOADER_STATS
DOWNLOADER_STATS
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index a71589d4a..3b9f4e11e 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -8,9 +8,10 @@ from __future__ import annotations
import warnings
from functools import wraps
+from itertools import islice
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
@@ -28,12 +29,21 @@ if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from scrapy import Spider
+ from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
class DownloaderMiddlewareManager(MiddlewareManager):
component_name = "downloader middleware"
+ def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:
+ super().__init__(*middlewares, crawler=crawler)
+ self._response_exceptions = (
+ crawler.settings.getbool("DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS")
+ if crawler is not None
+ else False
+ )
+
@classmethod
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
return build_component_list(
@@ -44,12 +54,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, "process_request"):
self.methods["process_request"].append(mw.process_request)
self._check_mw_method_spider_arg(mw.process_request)
- if hasattr(mw, "process_response"):
- self.methods["process_response"].appendleft(mw.process_response)
- self._check_mw_method_spider_arg(mw.process_response)
- if hasattr(mw, "process_exception"):
- self.methods["process_exception"].appendleft(mw.process_exception)
- self._check_mw_method_spider_arg(mw.process_exception)
+ process_response = getattr(mw, "process_response", None)
+ self.methods["process_response"].appendleft(process_response)
+ if process_response is not None:
+ self._check_mw_method_spider_arg(process_response)
+ process_exception = getattr(mw, "process_exception", None)
+ self.methods["process_exception"].appendleft(process_exception)
+ if process_exception is not None:
+ self._check_mw_method_spider_arg(process_exception)
def download(
self,
@@ -84,7 +96,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
await _process_pending_io()
# either returns a request or response (which we pass to process_response())
# or reraises the exception
- result = await self._process_exception(ex, request)
+ result, _ = await self._process_exception(ex, request)
return await self._process_response(result, request)
def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:
@@ -114,34 +126,96 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return await download_func(request)
async def _process_response(
- self, response: Response | Request, request: Request
+ self, response: Response | Request, request: Request, index: int = 0
) -> Response | Request:
if response is None:
raise TypeError("Received None in process_response")
- if isinstance(response, Request):
- return response
-
- for method in self.methods["process_response"]:
- assert method is not None
- response = await ensure_awaitable(
- self._handle_mw_method(method, request=request, response=response),
- _warn=global_object_name(method),
- )
- if not isinstance(response, (Response, Request)):
+ methods = self.methods["process_response"]
+ while index < len(methods):
+ if isinstance(response, Request):
+ return response
+ method = methods[index]
+ index += 1
+ if method is None:
+ continue
+ try:
+ result = await ensure_awaitable(
+ self._handle_mw_method(method, request=request, response=response),
+ _warn=global_object_name(method),
+ )
+ except Exception as ex:
+ if not self._response_exceptions:
+ # CloseSpider closes the spider either way, it never
+ # reaches process_exception().
+ if not isinstance(ex, CloseSpider):
+ self._warn_response_exception(method, ex, index)
+ raise
+ response, index = await self._process_exception(ex, request, index)
+ continue
+
+ if not isinstance(result, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, "
- f"got {type(response)}"
+ f"got {type(result)}"
)
- if isinstance(response, Request):
- return response
+ response = result
return response
+ def _warn_response_exception(
+ self, method: Callable[..., Any], exception: Exception, index: int
+ ) -> None:
+ """Warn that *exception*, raised by the process_response() *method* of a
+ downloader middleware, will go to the process_exception() methods from
+ *index* on once DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS is mandatory.
+
+ Warn about nothing when all those methods are Scrapy's own: they are
+ nothing for users to review, and RetryMiddleware, the one that acts on
+ these exceptions, still lets them reach the request errback once
+ retries are exhausted.
+ """
+ targets = [
+ global_object_name(target)
+ for target in islice(self.methods["process_exception"], index, None)
+ if target is not None and not target.__module__.startswith("scrapy.")
+ ]
+ if not targets:
+ return
+ warnings.warn(
+ f"{global_object_name(method)} raised"
+ f" {global_object_name(type(exception))}. In a future Scrapy version,"
+ " exceptions raised by the process_response() method of a downloader"
+ " middleware will be passed to the process_exception() method of the"
+ " downloader middlewares that have not processed the response yet,"
+ " instead of reaching the request errback directly. Before setting the"
+ " DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS setting to True to get"
+ " that behavior now, make sure that this exception is handled as"
+ " intended by the following methods, e.g. that an IgnoreRequest"
+ " exception raised to drop a response is not retried:"
+ f" {', '.join(targets)}.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
async def _process_exception(
- self, exception: Exception, request: Request | Response
- ) -> Response | Request:
- for method in self.methods["process_exception"]:
- assert method is not None
+ self, exception: Exception, request: Request, index: int = 0
+ ) -> tuple[Response | Request, int]:
+ """Let middlewares handle *exception*, starting at *index*.
+
+ Return the result of the middleware that handled it and the index of
+ the next middleware, or reraise *exception* if none handled it.
+ """
+ # CloseSpider asks the engine to close the spider, it is not a download
+ # error for middlewares to recover from.
+ if isinstance(exception, CloseSpider):
+ raise exception
+
+ methods = self.methods["process_exception"]
+ while index < len(methods):
+ method = methods[index]
+ index += 1
+ if method is None:
+ continue
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, exception=exception),
_warn=global_object_name(method),
@@ -152,5 +226,5 @@ class DownloaderMiddlewareManager(MiddlewareManager):
f"Request, got {type(response)}"
)
if response:
- return response
+ return response, index
raise exception
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index bfdda63ca..6c68fc65a 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -263,6 +263,14 @@ class Scraper:
await self.handle_spider_output_async(output, request, result)
return
+ if result.check(CloseSpider):
+ exc = result.value
+ assert isinstance(exc, CloseSpider) # typing
+ _schedule_coro(
+ self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
+ )
+ return
+
try:
# call the request errback with the downloader error
output = await self.call_spider_async(result, request)
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 674384a26..f01c8a1f9 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -6,10 +6,16 @@ from logging import getLogger
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import (
+ DecompressionError,
+ IgnoreRequest,
+ NotConfigured,
+ ScrapyDeprecationWarning,
+)
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._compression import (
+ _DECOMPRESSION_ERRORS,
_DecompressionMaxSizeExceeded,
_inflate,
_unbrotli,
@@ -104,6 +110,11 @@ class HttpCompressionMiddleware:
)
logger.warning(msg)
raise IgnoreRequest(msg) from e
+ except _DECOMPRESSION_ERRORS as e:
+ encodings = b", ".join(content_encoding).decode()
+ raise DecompressionError(
+ f"Could not decompress {response} ({encodings}): {e}"
+ ) from e
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 70d284188..db8b5d196 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -56,12 +56,16 @@ class DontCloseSpider(Exception):
class CloseSpider(Exception):
- """Raised from a :ref:`spider callback <topics-spiders>`, or while the
+ """Raised from a :ref:`spider callback <topics-spiders>`, a
+ :ref:`downloader middleware <topics-downloader-middleware>`, or while the
spider is starting, to request the spider to be closed/stopped.
.. versionchanged:: 2.18.0
Added support for raising it while the spider is starting.
+ .. versionchanged:: VERSION
+ Added support for raising it from a downloader middleware.
+
*reason* is a string with the reason for closing.
For example:
@@ -133,6 +137,15 @@ class ResponseDataLossError(Exception):
"""Indicates that Scrapy couldn't get a complete response."""
+class DecompressionError(Exception):
+ """Raised by
+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+ when a response body cannot be decompressed.
+
+ .. versionadded:: VERSION
+ """
+
+
class UnsupportedURLSchemeError(Exception):
"""Indicates that the URL scheme is not supported."""
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0c0216ed2..19a51adcd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -65,6 +65,7 @@ __all__ = [
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING",
"DOWNLOADER_MIDDLEWARES",
"DOWNLOADER_MIDDLEWARES_BASE",
+ "DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS",
"DOWNLOADER_STATS",
"DOWNLOAD_BIND_ADDRESS",
"DOWNLOAD_DELAY",
@@ -343,6 +344,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
"scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,
# Downloader side
}
+DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = False
DOWNLOADER_STATS = True
@@ -530,6 +532,7 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
RETRY_ENABLED = True
RETRY_EXCEPTIONS = [
"scrapy.exceptions.CannotResolveHostError",
+ "scrapy.exceptions.DecompressionError",
"scrapy.exceptions.DownloadConnectionRefusedError",
"scrapy.exceptions.DownloadFailedError",
"scrapy.exceptions.DownloadTimeoutError",
@@ -537,8 +540,6 @@ RETRY_EXCEPTIONS = [
"twisted.internet.error.ConnectionDone",
"twisted.internet.error.ConnectError",
"twisted.internet.error.ConnectionLost",
- # OSError is raised by the HttpCompression middleware when trying to
- # decompress an empty response
OSError,
"scrapy.core.downloader.handlers.http11.TunnelError",
]
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 0432a7231..be874b3df 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -84,4 +84,5 @@ DOWNLOAD_DELAY = 1
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Set settings whose default value is deprecated to a future-proof value
+DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = True
FEED_EXPORT_ENCODING = "utf-8"
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 1966b4f1c..2be0bd434 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -23,6 +23,15 @@ if TYPE_CHECKING:
_CHUNK_SIZE = 65536 # 64 KiB
+_DECOMPRESSION_ERRORS: tuple[type[Exception], ...] = (
+ EOFError,
+ OSError,
+ brotli.error,
+ struct.error,
+ zlib.error,
+ zstd.ZstdError,
+)
+
class _DecompressionMaxSizeExceeded(ValueError):
def __init__(self, decompressed_size: int, max_size: int) -> None:
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 018b82f23..da0a08b7b 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -67,6 +67,16 @@ if TYPE_CHECKING:
from tests.mockserver.http import MockServer
+class CloseSpiderProcessRequestMiddleware:
+ def process_request(self, request):
+ raise CloseSpider("my_reason")
+
+
+class CloseSpiderProcessResponseMiddleware:
+ def process_response(self, request, response):
+ raise CloseSpider("my_reason")
+
+
class TestCrawl:
@coroutine_test
async def test_follow_all(self, mockserver: MockServer) -> None:
@@ -1025,3 +1035,22 @@ class TestCrawlSpider:
await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb)
assert "Closing spider (my_reason)" in caplog.text
assert "Spider error processing" not in caplog.text
+
+ @pytest.mark.parametrize(
+ "middleware",
+ [CloseSpiderProcessRequestMiddleware, CloseSpiderProcessResponseMiddleware],
+ )
+ @coroutine_test
+ async def test_raise_closespider_downloader_middleware(
+ self,
+ caplog: pytest.LogCaptureFixture,
+ mockserver: MockServer,
+ middleware: type,
+ ) -> None:
+ crawler = get_crawler(
+ SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {middleware: 1}}
+ )
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async(seed=mockserver.url("/"))
+ assert "Closing spider (my_reason)" in caplog.text
+ assert "Error downloading" not in caplog.text
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 46cb6b920..163aa70d4 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -2,15 +2,21 @@ from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
-from gzip import BadGzipFile
-from typing import TYPE_CHECKING
+from gzip import compress
+from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
from twisted.internet.defer import Deferred, succeed
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
-from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
+from scrapy.exceptions import (
+ CloseSpider,
+ DecompressionError,
+ IgnoreRequest,
+ ScrapyDeprecationWarning,
+ _InvalidOutput,
+)
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
@@ -23,8 +29,37 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+def _invalid_gzipped_response(request: Request) -> Response:
+ body = b"<p>You are being redirected</p>"
+ return Response(
+ request.url,
+ status=200,
+ body=body,
+ headers={
+ "Content-Length": str(len(body)),
+ "Content-Type": "text/html",
+ "Content-Encoding": "gzip",
+ "Location": "http://example.com/login",
+ },
+ )
+
+
+def _truncated_gzipped_response(request: Request) -> Response:
+ body = compress(b"<p>You are being redirected</p>")[:10]
+ return Response(
+ request.url,
+ status=200,
+ body=body,
+ headers={
+ "Content-Length": str(len(body)),
+ "Content-Type": "text/html",
+ "Content-Encoding": "gzip",
+ },
+ )
+
+
class TestManagerBase:
- settings_dict = None
+ settings_dict: dict[str, Any] | None = None
# should be a fixture but async fixtures that use Futures are problematic with pytest-twisted
@asynccontextmanager
@@ -117,19 +152,8 @@ class TestDefaults(TestManagerBase):
@coroutine_test
async def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com")
- body = b"<p>You are being redirected</p>"
- resp = Response(
- req.url,
- status=200,
- body=body,
- headers={
- "Content-Length": str(len(body)),
- "Content-Type": "text/html",
- "Content-Encoding": "gzip",
- "Location": "http://example.com/login",
- },
- )
- with pytest.raises(BadGzipFile):
+ resp = _invalid_gzipped_response(req)
+ with pytest.raises(DecompressionError):
async with self.get_mwman() as mwman:
await self._download(mwman, req, resp)
@@ -185,6 +209,219 @@ class TestResponseFromProcessException(TestManagerBase):
]
+class TestResponseExceptions(TestManagerBase):
+ """Tests exceptions from process_response reaching process_exception."""
+
+ settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": True}
+
+ @coroutine_test
+ async def test_request_from_process_exception(self):
+ req = Request("http://example.com/index.html")
+ retry_req = Request("http://example.com/index.html")
+ calls = []
+
+ class OuterMiddleware:
+ def process_response(self, request, response):
+ calls.append("outer.process_response")
+ return response
+
+ def process_exception(self, request, exception):
+ calls.append("outer.process_exception")
+ return retry_req
+
+ class InnerMiddleware:
+ def process_response(self, request, response):
+ calls.append("inner.process_response")
+ raise ValueError("test")
+
+ def process_exception(self, request, exception):
+ calls.append("inner.process_exception")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(OuterMiddleware())
+ mwman._add_middleware(InnerMiddleware())
+ result = await self._download(mwman, req)
+ assert result is retry_req
+ assert calls == ["inner.process_response", "outer.process_exception"]
+
+ @coroutine_test
+ async def test_response_from_process_exception(self):
+ req = Request("http://example.com/index.html")
+ recovered = Response("http://example.com/index.html")
+ calls = []
+
+ class OuterMiddleware:
+ def process_response(self, request, response):
+ calls.append("outer.process_response")
+ return response
+
+ class MiddleMiddleware:
+ def process_response(self, request, response):
+ calls.append("middle.process_response")
+ return response
+
+ def process_exception(self, request, exception):
+ calls.append("middle.process_exception")
+ return recovered
+
+ class InnerMiddleware:
+ def process_response(self, request, response):
+ calls.append("inner.process_response")
+ raise ValueError("test")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(OuterMiddleware())
+ mwman._add_middleware(MiddleMiddleware())
+ mwman._add_middleware(InnerMiddleware())
+ result = await self._download(mwman, req)
+ assert result is recovered
+ assert calls == [
+ "inner.process_response",
+ "middle.process_exception",
+ "outer.process_response",
+ ]
+
+ @coroutine_test
+ async def test_processed_middleware_not_called(self):
+ req = Request("http://example.com/index.html")
+ calls = []
+
+ class RaisingMiddleware:
+ def process_response(self, request, response):
+ calls.append("raising.process_response")
+ raise ValueError("test")
+
+ class InnerMiddleware:
+ def process_response(self, request, response):
+ calls.append("inner.process_response")
+ return response
+
+ def process_exception(self, request, exception):
+ calls.append("inner.process_exception")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(RaisingMiddleware())
+ mwman._add_middleware(InnerMiddleware())
+ with pytest.raises(ValueError, match="test"):
+ await self._download(mwman, req)
+ assert calls == ["inner.process_response", "raising.process_response"]
+
+ @coroutine_test
+ async def test_unhandled(self):
+ req = Request("http://example.com/index.html")
+
+ class RaisingMiddleware:
+ def process_response(self, request, response):
+ raise ValueError("test")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(RaisingMiddleware())
+ with pytest.raises(ValueError, match="test"):
+ await self._download(mwman, req)
+
+ @pytest.mark.parametrize(
+ "response_func", [_invalid_gzipped_response, _truncated_gzipped_response]
+ )
+ @coroutine_test
+ async def test_retry(self, response_func):
+ """Decompression failures are retried, as RETRY_EXCEPTIONS promises."""
+ req = Request("http://example.com")
+ async with self.get_mwman() as mwman:
+ result = await self._download(mwman, req, response_func(req))
+ assert isinstance(result, Request)
+ assert result.url == req.url
+
+ @coroutine_test
+ async def test_close_spider(self):
+ req = Request("http://example.com/index.html")
+ calls = []
+
+ class OuterMiddleware:
+ def process_exception(self, request, exception):
+ calls.append("outer.process_exception")
+
+ class InnerMiddleware:
+ def process_response(self, request, response):
+ raise CloseSpider("test")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(OuterMiddleware())
+ mwman._add_middleware(InnerMiddleware())
+ with pytest.raises(CloseSpider):
+ await self._download(mwman, req)
+ assert not calls
+
+ @coroutine_test
+ async def test_close_spider_from_process_request(self):
+ req = Request("http://example.com/index.html")
+ calls = []
+
+ class Middleware:
+ def process_request(self, request):
+ raise CloseSpider("test")
+
+ def process_exception(self, request, exception):
+ calls.append("process_exception")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(Middleware())
+ with pytest.raises(CloseSpider):
+ await self._download(mwman, req)
+ assert not calls
+
+
+class TestResponseExceptionsDisabled(TestManagerBase):
+ """Tests the deprecated behavior, which the default value still gives."""
+
+ @pytest.mark.parametrize("exception", [ValueError, IgnoreRequest])
+ @coroutine_test
+ async def test_not_passed_to_process_exception(self, exception):
+ req = Request("http://example.com/index.html")
+ calls = []
+
+ class OuterMiddleware:
+ def process_exception(self, request, exception):
+ calls.append("outer.process_exception")
+
+ class InnerMiddleware:
+ def process_response(self, request, response):
+ raise exception("test")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(OuterMiddleware())
+ mwman._add_middleware(InnerMiddleware())
+ with (
+ pytest.raises(exception, match="test"),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match="OuterMiddleware.process_exception",
+ ),
+ ):
+ await self._download(mwman, req)
+ assert not calls
+
+ @coroutine_test
+ async def test_only_built_in_process_exception(self):
+ """Nothing is warned about when only Scrapy's own process_exception()
+ methods would get the exception."""
+ req = Request("http://example.com/index.html")
+
+ class RaisingMiddleware:
+ def process_response(self, request, response):
+ raise ValueError("test")
+
+ async with self.get_mwman() as mwman:
+ mwman._add_middleware(RaisingMiddleware())
+ with pytest.raises(ValueError, match="test"):
+ await self._download(mwman, req)
+
+
+class TestResponseExceptionsDisabledExplicitly(TestResponseExceptionsDisabled):
+ """Asking for the deprecated behavior does not silence the warning."""
+
+ settings_dict = {"DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS": False}
+
+
class TestInvalidOutput(TestManagerBase):
@coroutine_test
async def test_invalid_process_request(self):
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 285982c46..96194ee86 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -13,7 +13,12 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import (
+ DecompressionError,
+ IgnoreRequest,
+ NotConfigured,
+ ScrapyDeprecationWarning,
+)
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -809,3 +814,22 @@ class TestHttpCompression:
continue
resp = self._get_truncated_response(check_key)
assert len(resp.body) == 0
+
+ @pytest.mark.parametrize(
+ ("body", "encoding"),
+ [
+ (b"not compressed", "gzip"),
+ (b"\x1f\x8b\x08\x00", "gzip"),
+ (b"\x1f\x8b\x08" + b"\x00" * 7 + b"\xff" * 20, "gzip"),
+ (b"not compressed", "deflate"),
+ (b"not compressed", "br"),
+ (b"not compressed", "zstd"),
+ ],
+ )
+ def test_process_response_invalid(self, body, encoding):
+ request = Request("http://example.com")
+ response = Response(
+ "http://example.com", body=body, headers={"Content-Encoding": encoding}
+ )
+ with pytest.raises(DecompressionError):
+ self.mw.process_response(request, response)