Scrapy, a fast high-level web crawling & scraping framework for Python.
Log ignored responses exceeding DOWNLOAD_MAXSIZE (#7742)
Mridankan Mandal1ddf024b8944
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 6ca04a50e..13a432476 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -123,12 +123,14 @@ class HttpCompressionMiddleware:response.body, content_encoding, max_size)except _DecompressionMaxSizeExceeded as e:- raise IgnoreRequest(+ msg = (f"Ignored response {response} because its body "f"({len(response.body)} B compressed, "f"{e.decompressed_size} B decompressed so far) exceeded "f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."- ) from e+ )+ logger.warning(msg)+ raise IgnoreRequest(msg) from eif len(response.body) < warn_size <= len(decoded_body):logger.warning(f"{response} body size after decompression "diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 5c4085657..311aff7df 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -558,6 +558,30 @@ class TestHttpCompression:self._test_compression_bomb_setting("zstd")+ def test_compression_bomb_setting_logs_warning(self, caplog):+ settings = {"DOWNLOAD_MAXSIZE": 1_000_000}+ crawler = get_crawler(Spider, settings_dict=settings)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)++ response = self._getresponse("bomb-gzip") # 11_511_612 B+ caplog.clear()+ with (+ caplog.at_level(+ WARNING, logger="scrapy.downloadermiddlewares.httpcompression"+ ),+ pytest.raises(IgnoreRequest) as exc_info,+ ):+ mw.process_response(response.request, response)+ assert caplog.record_tuples == [+ (+ "scrapy.downloadermiddlewares.httpcompression",+ WARNING,+ str(exc_info.value),+ )+ ]+def _test_compression_bomb_spider_attr(self, compression_id):class DownloadMaxSizeSpider(Spider):download_maxsize = 1_000_000