Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Log ignored responses exceeding DOWNLOAD_MAXSIZE (#7742)

Mridankan Mandal1ddf024b8944
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 6ca04a50e..13a432476 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -123,12 +123,14 @@ class HttpCompressionMiddleware:
response.body, content_encoding, max_size
)
except _DecompressionMaxSizeExceeded as e:
- raise IgnoreRequest(
+ msg = (
f"Ignored response {response} because its body "
f"({len(response.body)} B compressed, "
f"{e.decompressed_size} B decompressed so far) exceeded "
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
- ) from e
+ )
+ logger.warning(msg)
+ raise IgnoreRequest(msg) from e
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 5c4085657..311aff7df 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -558,6 +558,30 @@ class TestHttpCompression:
self._test_compression_bomb_setting("zstd")
+ def test_compression_bomb_setting_logs_warning(self, caplog):
+ settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
+ crawler = get_crawler(Spider, settings_dict=settings)
+ spider = crawler._create_spider("scrapytest.org")
+ mw = HttpCompressionMiddleware.from_crawler(crawler)
+ mw.open_spider(spider)
+
+ response = self._getresponse("bomb-gzip") # 11_511_612 B
+ caplog.clear()
+ with (
+ caplog.at_level(
+ WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+ ),
+ pytest.raises(IgnoreRequest) as exc_info,
+ ):
+ mw.process_response(response.request, response)
+ assert caplog.record_tuples == [
+ (
+ "scrapy.downloadermiddlewares.httpcompression",
+ WARNING,
+ str(exc_info.value),
+ )
+ ]
+
def _test_compression_bomb_spider_attr(self, compression_id):
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000