Scrapy, a fast high-level web crawling & scraping framework for Python.
Protect against gzip bombs
Adrián Chaves6969041c5f68
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex ead426951..5dd67ea87 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -2,9 +2,10 @@ import ioimport warningsimport zlib-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes+from scrapy.utils._compression import _DecompressionMaxSizeExceededfrom scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.gz import gunzip@@ -29,24 +30,26 @@ class HttpCompressionMiddleware:"""This middleware allows compressed (gzip, deflate) traffic to besent/received from web sites"""- def __init__(self, stats=None):- self.stats = stats+ def __init__(self, crawler=None):+ self.stats = crawler.stats+ self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool("COMPRESSION_ENABLED"):raise NotConfiguredtry:- return cls(stats=crawler.stats)+ return cls(crawler=crawler)except TypeError:warnings.warn("HttpCompressionMiddleware subclasses must either modify "- "their '__init__' method to support a 'stats' parameter or "- "reimplement the 'from_crawler' method.",+ "their '__init__' method to support a 'crawler' parameter or "+ "reimplement their 'from_crawler' method.",ScrapyDeprecationWarning,)result = cls()result.stats = crawler.stats+ result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")return resultdef process_request(self, request, spider):@@ -59,7 +62,14 @@ class HttpCompressionMiddleware:content_encoding = response.headers.getlist("Content-Encoding")if content_encoding:encoding = content_encoding.pop()- decoded_body = self._decode(response.body, encoding.lower())+ try:+ decoded_body = self._decode(response.body, encoding.lower())+ except _DecompressionMaxSizeExceeded:+ raise IgnoreRequest(+ f"Ignored response {response} because its body "+ f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE "+ f"({self._max_size}B) during decompression."+ )if self.stats:self.stats.inc_value("httpcompression/response_bytes",@@ -85,7 +95,7 @@ class HttpCompressionMiddleware:def _decode(self, body, encoding):if encoding == b"gzip" or encoding == b"x-gzip":- body = gunzip(body)+ body = gunzip(body, max_size=self._max_size)if encoding == b"deflate":try:diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pynew file mode 100644index 000000000..e726a70f5--- /dev/null+++ b/scrapy/utils/_compression.py@@ -0,0 +1,2 @@+class _DecompressionMaxSizeExceeded(ValueError):+ passdiff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex c7f74030e..cd5059a5c 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -5,8 +5,10 @@ from typing import Listfrom scrapy.http import Response+from ._compression import _DecompressionMaxSizeExceeded-def gunzip(data: bytes) -> bytes:++def gunzip(data: bytes, max_size: int = 0) -> bytes:"""Gunzip the given data and return as much data as possible.This is resilient to CRC checksum errors.@@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes:f = GzipFile(fileobj=BytesIO(data))output_list: List[bytes] = []chunk = b"."+ decompressed_size = 0while chunk:try:chunk = f.read1(8196)- output_list.append(chunk)except (OSError, EOFError, struct.error):# complete only if there is some data, otherwise re-raise# see issue 87 about catching struct.error@@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes:if output_list:breakraise+ decompressed_size += len(chunk)+ if max_size and decompressed_size > max_size:+ raise _DecompressionMaxSizeExceeded(+ f"The number of bytes decompressed so far "+ f"({decompressed_size}B) exceed the specified maximum "+ f"({max_size}B)."+ )+ output_list.append(chunk)return b"".join(output_list)diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.binnew file mode 100644index 000000000..64aa0c369Binary files /dev/null and b/tests/sample_data/compressed/bomb-gzip.bin differdiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 9dad056de..f834e78f5 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (ACCEPTED_ENCODINGS,HttpCompressionMiddleware,)-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider@@ -35,12 +35,24 @@ FORMAT = {"html-zstd-streaming-no-content-size.bin","zstd",),+ **{+ f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)+ for format_id in (+ # "br",+ "gzip", # 27 988 → 11 511 612+ # "deflate",+ # "zstd",+ )+ },}class HttpCompressionTest(TestCase):def setUp(self):- self.crawler = get_crawler(Spider)+ settings = {+ "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests.+ }+ self.crawler = get_crawler(Spider, settings_dict=settings)self.spider = self.crawler._create_spider("scrapytest.org")self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)self.crawler.stats.open_spider(self.spider)@@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase):self.assertStatsEqual("httpcompression/response_count", None)self.assertStatsEqual("httpcompression/response_bytes", None)+ def _test_compression_bomb(self, compression_id):+ response = self._getresponse(f"bomb-{compression_id}")+ self.assertRaises(+ IgnoreRequest,+ self.mw.process_response,+ response.request,+ response,+ self.spider,+ )++ def test_compression_bomb_gzip(self):+ self._test_compression_bomb("gzip")+class HttpCompressionSubclassTest(TestCase):def test_init_missing_stats(self):