Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Protect against brotli bombs

Adrián Chavesfba167c5e1f3
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 8cec87c47..91748e57e 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -4,25 +4,25 @@ import warnings
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
-from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate
+from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
try:
- import brotli
-
- ACCEPTED_ENCODINGS.append(b"br")
+ import brotli # noqa: F401
except ImportError:
pass
+else:
+ ACCEPTED_ENCODINGS.append(b"br")
try:
import zstandard
-
- ACCEPTED_ENCODINGS.append(b"zstd")
except ImportError:
pass
+else:
+ ACCEPTED_ENCODINGS.append(b"zstd")
class HttpCompressionMiddleware:
@@ -100,7 +100,7 @@ class HttpCompressionMiddleware:
if encoding == b"deflate":
return _inflate(body, max_size=self._max_size)
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
- return brotli.decompress(body)
+ return _unbrotli(body, max_size=self._max_size)
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
# Using its streaming API since its simple API could handle only cases
# where there is content size data embedded in the frame
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 34bf2e4f7..9a32ce4f0 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -2,6 +2,11 @@ import zlib
from io import BytesIO
from typing import List
+try:
+ import brotli
+except ImportError:
+ pass
+
class _DecompressionMaxSizeExceeded(ValueError):
pass
@@ -39,3 +44,24 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
)
output_list.append(output_chunk)
return b"".join(output_list)
+
+
+def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
+ decompressor = brotli.Decompressor()
+ input_stream = BytesIO(data)
+ output_list: List[bytes] = []
+ output_chunk = b"."
+ decompressed_size = 0
+ CHUNK_SIZE = 8196
+ while output_chunk:
+ input_chunk = input_stream.read(CHUNK_SIZE)
+ output_chunk = decompressor.process(input_chunk)
+ decompressed_size += len(output_chunk)
+ if max_size and decompressed_size > max_size:
+ raise _DecompressionMaxSizeExceeded(
+ f"The number of bytes decompressed so far "
+ f"({decompressed_size}B) exceed the specified maximum "
+ f"({max_size}B)."
+ )
+ output_list.append(output_chunk)
+ return b"".join(output_list)
diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.bin
new file mode 100644
index 000000000..50059866f
--- /dev/null
+++ b/tests/sample_data/compressed/bomb-br.bin
@@ -0,0 +1,2 @@
+�;�����nުVp SmoY2��
+�(�)-д=_o
\ No newline at end of file
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 3af8202cc..8858916bc 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -38,7 +38,7 @@ FORMAT = {
**{
f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
for format_id in (
- # "br",
+ "br", # 34 → 11 511 612
"deflate", # 27 968 → 11 511 612
"gzip", # 27 988 → 11 511 612
# "zstd",
@@ -383,6 +383,13 @@ class HttpCompressionTest(TestCase):
self.spider,
)
+ def test_compression_bomb_br(self):
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ raise SkipTest("no brotli")
+ self._test_compression_bomb("br")
+
def test_compression_bomb_deflate(self):
self._test_compression_bomb("deflate")