Scrapy, a fast high-level web crawling & scraping framework for Python.
Protect against brotli bombs
Adrián Chavesfba167c5e1f3
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 8cec87c47..91748e57e 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -4,25 +4,25 @@ import warningsfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes-from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate+from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotlifrom scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.gz import gunzipACCEPTED_ENCODINGS = [b"gzip", b"deflate"]try:- import brotli-- ACCEPTED_ENCODINGS.append(b"br")+ import brotli # noqa: F401except ImportError:pass+else:+ ACCEPTED_ENCODINGS.append(b"br")try:import zstandard-- ACCEPTED_ENCODINGS.append(b"zstd")except ImportError:pass+else:+ ACCEPTED_ENCODINGS.append(b"zstd")class HttpCompressionMiddleware:@@ -100,7 +100,7 @@ class HttpCompressionMiddleware:if encoding == b"deflate":return _inflate(body, max_size=self._max_size)if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:- return brotli.decompress(body)+ return _unbrotli(body, max_size=self._max_size)if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:# Using its streaming API since its simple API could handle only cases# where there is content size data embedded in the framediff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 34bf2e4f7..9a32ce4f0 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -2,6 +2,11 @@ import zlibfrom io import BytesIOfrom typing import List+try:+ import brotli+except ImportError:+ pass+class _DecompressionMaxSizeExceeded(ValueError):pass@@ -39,3 +44,24 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:)output_list.append(output_chunk)return b"".join(output_list)+++def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:+ decompressor = brotli.Decompressor()+ input_stream = BytesIO(data)+ output_list: List[bytes] = []+ output_chunk = b"."+ decompressed_size = 0+ CHUNK_SIZE = 8196+ while output_chunk:+ input_chunk = input_stream.read(CHUNK_SIZE)+ output_chunk = decompressor.process(input_chunk)+ decompressed_size += len(output_chunk)+ if max_size and decompressed_size > max_size:+ raise _DecompressionMaxSizeExceeded(+ f"The number of bytes decompressed so far "+ f"({decompressed_size}B) exceed the specified maximum "+ f"({max_size}B)."+ )+ output_list.append(output_chunk)+ return b"".join(output_list)diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.binnew file mode 100644index 000000000..50059866f--- /dev/null+++ b/tests/sample_data/compressed/bomb-br.bin@@ -0,0 +1,2 @@+�;�����nުVp SmoY2��+�(�)-д=_o\ No newline at end of filediff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 3af8202cc..8858916bc 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -38,7 +38,7 @@ FORMAT = {**{f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)for format_id in (- # "br",+ "br", # 34 → 11 511 612"deflate", # 27 968 → 11 511 612"gzip", # 27 988 → 11 511 612# "zstd",@@ -383,6 +383,13 @@ class HttpCompressionTest(TestCase):self.spider,)+ def test_compression_bomb_br(self):+ try:+ import brotli # noqa: F401+ except ImportError:+ raise SkipTest("no brotli")+ self._test_compression_bomb("br")+def test_compression_bomb_deflate(self):self._test_compression_bomb("deflate")