Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Protect against gzip bombs

Adrián Chaves6969041c5f68
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index ead426951..5dd67ea87 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -2,9 +2,10 @@ import io
import warnings
import zlib
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
+from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
@@ -29,24 +30,26 @@ class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
- def __init__(self, stats=None):
- self.stats = stats
+ def __init__(self, crawler=None):
+ self.stats = crawler.stats
+ self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
try:
- return cls(stats=crawler.stats)
+ return cls(crawler=crawler)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
- "their '__init__' method to support a 'stats' parameter or "
- "reimplement the 'from_crawler' method.",
+ "their '__init__' method to support a 'crawler' parameter or "
+ "reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
result = cls()
result.stats = crawler.stats
+ result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
return result
def process_request(self, request, spider):
@@ -59,7 +62,14 @@ class HttpCompressionMiddleware:
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
encoding = content_encoding.pop()
- decoded_body = self._decode(response.body, encoding.lower())
+ try:
+ decoded_body = self._decode(response.body, encoding.lower())
+ except _DecompressionMaxSizeExceeded:
+ raise IgnoreRequest(
+ f"Ignored response {response} because its body "
+ f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE "
+ f"({self._max_size}B) during decompression."
+ )
if self.stats:
self.stats.inc_value(
"httpcompression/response_bytes",
@@ -85,7 +95,7 @@ class HttpCompressionMiddleware:
def _decode(self, body, encoding):
if encoding == b"gzip" or encoding == b"x-gzip":
- body = gunzip(body)
+ body = gunzip(body, max_size=self._max_size)
if encoding == b"deflate":
try:
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
new file mode 100644
index 000000000..e726a70f5
--- /dev/null
+++ b/scrapy/utils/_compression.py
@@ -0,0 +1,2 @@
+class _DecompressionMaxSizeExceeded(ValueError):
+ pass
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index c7f74030e..cd5059a5c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -5,8 +5,10 @@ from typing import List
from scrapy.http import Response
+from ._compression import _DecompressionMaxSizeExceeded
-def gunzip(data: bytes) -> bytes:
+
+def gunzip(data: bytes, max_size: int = 0) -> bytes:
"""Gunzip the given data and return as much data as possible.
This is resilient to CRC checksum errors.
@@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes:
f = GzipFile(fileobj=BytesIO(data))
output_list: List[bytes] = []
chunk = b"."
+ decompressed_size = 0
while chunk:
try:
chunk = f.read1(8196)
- output_list.append(chunk)
except (OSError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error
@@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes:
if output_list:
break
raise
+ decompressed_size += len(chunk)
+ if max_size and decompressed_size > max_size:
+ raise _DecompressionMaxSizeExceeded(
+ f"The number of bytes decompressed so far "
+ f"({decompressed_size}B) exceed the specified maximum "
+ f"({max_size}B)."
+ )
+ output_list.append(chunk)
return b"".join(output_list)
diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin
new file mode 100644
index 000000000..64aa0c369
Binary files /dev/null and b/tests/sample_data/compressed/bomb-gzip.bin differ
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 9dad056de..f834e78f5 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -35,12 +35,24 @@ FORMAT = {
"html-zstd-streaming-no-content-size.bin",
"zstd",
),
+ **{
+ f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
+ for format_id in (
+ # "br",
+ "gzip", # 27 988 → 11 511 612
+ # "deflate",
+ # "zstd",
+ )
+ },
}
class HttpCompressionTest(TestCase):
def setUp(self):
- self.crawler = get_crawler(Spider)
+ settings = {
+ "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests.
+ }
+ self.crawler = get_crawler(Spider, settings_dict=settings)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider(self.spider)
@@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase):
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
+ def _test_compression_bomb(self, compression_id):
+ response = self._getresponse(f"bomb-{compression_id}")
+ self.assertRaises(
+ IgnoreRequest,
+ self.mw.process_response,
+ response.request,
+ response,
+ self.spider,
+ )
+
+ def test_compression_bomb_gzip(self):
+ self._test_compression_bomb("gzip")
+
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):