Scrapy, a fast high-level web crawling & scraping framework for Python.
Mitigate brotli and deflate decompression bombs DoS (#7134)
Rui Xi14737e91edc5
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 4aa76e6c6..1f1ef77ca 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -62,6 +62,9 @@ jobs:- python-version: "3.13"env:TOXENV: botocore+ - python-version: "3.13"+ env:+ TOXENV: mitmproxysteps:- uses: actions/checkout@v5diff --git a/conftest.py b/conftest.pyindex ece57eecc..0dd9d1ee0 100644--- a/conftest.py+++ b/conftest.py@@ -117,6 +117,16 @@ def requires_boto3(request):pytest.skip("boto3 is not installed")[email protected](autouse=True)+def requires_mitmproxy(request):+ if not request.node.get_closest_marker("requires_mitmproxy"):+ return+ try:+ import mitmproxy # noqa: F401, PLC0415+ except ImportError:+ pytest.skip("mitmproxy is not installed")++def pytest_configure(config):if config.getoption("--reactor") == "asyncio":# Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.diff --git a/pyproject.toml b/pyproject.tomlindex 9da2309db..8f780e45e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -232,6 +232,7 @@ markers = ["requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)","requires_boto3: marks tests that need botocore and boto3",+ "requires_mitmproxy: marks tests that need mitmproxy",]filterwarnings = ["ignore::DeprecationWarning:twisted.web.static"diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex f729649a0..fd508f759 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -31,14 +31,20 @@ logger = getLogger(__name__)ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"]try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401+ import brotliexcept ImportError:passelse:- ACCEPTED_ENCODINGS.append(b"br")+ try:+ brotli.Decompressor.can_accept_more_data+ except AttributeError: # pragma: no cover+ warnings.warn(+ "You have brotli installed. But 'br' encoding support now requires "+ "brotli version >= 1.2.0. Please upgrade brotli version to make Scrapy "+ "decode 'br' encoded responses.",+ )+ else:+ ACCEPTED_ENCODINGS.append(b"br")try:import zstandard # noqa: F401@@ -116,13 +122,13 @@ class HttpCompressionMiddleware:decoded_body, content_encoding = self._handle_encoding(response.body, content_encoding, max_size)- except _DecompressionMaxSizeExceeded:+ except _DecompressionMaxSizeExceeded as e:raise IgnoreRequest(f"Ignored response {response} because its body "- f"({len(response.body)} B compressed) exceeded "- f"DOWNLOAD_MAXSIZE ({max_size} B) during "- f"decompression."- )+ f"({len(response.body)} B compressed, "+ f"{e.decompressed_size} B decompressed so far) exceeded "+ f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."+ ) from eif len(response.body) < warn_size <= len(decoded_body):logger.warning(f"{response} body size after decompression "@@ -202,7 +208,7 @@ class HttpCompressionMiddleware:f"from unsupported encoding(s) '{encodings_str}'.")if b"br" in encodings:- msg += " You need to install brotli or brotlicffi to decode 'br'."+ msg += " You need to install brotli >= 1.2.0 to decode 'br'."if b"zstd" in encodings:msg += " You need to install zstandard to decode 'zstd'."logger.warning(msg)diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 6b09f36ff..0a8191cc6 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -1,42 +1,9 @@import contextlibimport zlibfrom io import BytesIO-from warnings import warn--from scrapy.exceptions import ScrapyDeprecationWarning--try:- try:- import brotli- except ImportError:- import brotlicffi as brotli-except ImportError:- pass-else:- try:- brotli.Decompressor.process- except AttributeError:- warn(- (- "You have brotlipy installed, and Scrapy will use it, but "- "Scrapy support for brotlipy is deprecated and will stop "- "working in a future version of Scrapy. brotlipy itself is "- "deprecated, it has been superseded by brotlicffi. Please, "- "uninstall brotlipy and install brotli or brotlicffi instead. "- "brotlipy has the same import name as brotli, so keeping both "- "installed is strongly discouraged."- ),- ScrapyDeprecationWarning,- )-- def _brotli_decompress(decompressor, data):- return decompressor.decompress(data)-- else:-- def _brotli_decompress(decompressor, data):- return decompressor.process(data)+with contextlib.suppress(ImportError):+ import brotliwith contextlib.suppress(ImportError):import zstandard@@ -46,62 +13,64 @@ _CHUNK_SIZE = 65536 # 64 KiBclass _DecompressionMaxSizeExceeded(ValueError):- pass+ def __init__(self, decompressed_size: int, max_size: int) -> None:+ self.decompressed_size = decompressed_size+ self.max_size = max_size++ def __str__(self) -> str:+ return (+ f"The number of bytes decompressed so far "+ f"({self.decompressed_size} B) exceeded the specified maximum "+ f"({self.max_size} B)."+ )+++def _check_max_size(decompressed_size: int, max_size: int) -> None:+ if max_size and decompressed_size > max_size:+ raise _DecompressionMaxSizeExceeded(decompressed_size, max_size)def _inflate(data: bytes, *, max_size: int = 0) -> bytes:decompressor = zlib.decompressobj()- raw_decompressor = zlib.decompressobj(wbits=-15)- input_stream = BytesIO(data)+ try:+ first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE)+ except zlib.error:+ # to work with raw deflate content that may be sent by microsoft servers.+ decompressor = zlib.decompressobj(wbits=-15)+ first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE)+ decompressed_size = len(first_chunk)+ _check_max_size(decompressed_size, max_size)output_stream = BytesIO()- output_chunk = b"."- decompressed_size = 0- while output_chunk:- input_chunk = input_stream.read(_CHUNK_SIZE)- try:- output_chunk = decompressor.decompress(input_chunk)- except zlib.error:- if decompressor != raw_decompressor:- # ugly hack to work with raw deflate content that may- # be sent by microsoft servers. For more information, see:- # http://carsten.codimi.de/gzip.yaws/- # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx- # http://www.gzip.org/zlib/zlib_faq.html#faq38- decompressor = raw_decompressor- output_chunk = decompressor.decompress(input_chunk)- else:- raise+ output_stream.write(first_chunk)+ while decompressor.unconsumed_tail:+ output_chunk = decompressor.decompress(+ decompressor.unconsumed_tail, max_length=_CHUNK_SIZE+ )decompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ if tail := decompressor.flush():+ decompressed_size += len(tail)+ _check_max_size(decompressed_size, max_size)+ output_stream.write(tail)+ return output_stream.getvalue()def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:decompressor = brotli.Decompressor()- input_stream = BytesIO(data)+ first_chunk = decompressor.process(data, output_buffer_limit=_CHUNK_SIZE)+ decompressed_size = len(first_chunk)+ _check_max_size(decompressed_size, max_size)output_stream = BytesIO()- output_chunk = b"."- decompressed_size = 0- while output_chunk:- input_chunk = input_stream.read(_CHUNK_SIZE)- output_chunk = _brotli_decompress(decompressor, input_chunk)+ output_stream.write(first_chunk)+ while not decompressor.is_finished():+ output_chunk = decompressor.process(b"", output_buffer_limit=_CHUNK_SIZE)+ if not output_chunk:+ breakdecompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:@@ -113,12 +82,6 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:while output_chunk:output_chunk = stream_reader.read(_CHUNK_SIZE)decompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex 85324361c..5f95bccbd 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -5,7 +5,7 @@ from gzip import GzipFilefrom io import BytesIOfrom typing import TYPE_CHECKING-from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded+from ._compression import _CHUNK_SIZE, _check_max_sizeif TYPE_CHECKING:from scrapy.http import Response@@ -31,15 +31,9 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:breakraisedecompressed_size += len(chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()def gzip_magic_number(response: Response) -> bool:diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 65ba2e7e2..2080b3806 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -52,11 +52,10 @@ FORMAT = {def _skip_if_no_br() -> None:try:- try:- import brotli # noqa: F401,PLC0415- except ImportError:- import brotlicffi # noqa: F401,PLC0415- except ImportError:+ import brotli # noqa: PLC0415++ brotli.Decompressor.can_accept_more_data+ except (ImportError, AttributeError):pytest.skip("no brotli support")@@ -153,14 +152,9 @@ class TestHttpCompression:def test_process_response_br_unsupported(self):try:- try:- import brotli # noqa: F401,PLC0415-- pytest.skip("Requires not having brotli support")- except ImportError:- import brotlicffi # noqa: F401,PLC0415+ import brotli # noqa: F401,PLC0415- pytest.skip("Requires not having brotli support")+ pytest.skip("Requires not having brotli support")except ImportError:passresponse = self._getresponse("br")@@ -179,7 +173,7 @@ class TestHttpCompression:("HttpCompressionMiddleware cannot decode the response for"" http://scrapytest.org/ from unsupported encoding(s) 'br'."- " You need to install brotli or brotlicffi to decode 'br'."+ " You need to install brotli >= 1.2.0 to decode 'br'."),),)@@ -511,15 +505,16 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", None)def _test_compression_bomb_setting(self, compression_id):- settings = {"DOWNLOAD_MAXSIZE": 10_000_000}+ settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)spider = crawler._create_spider("scrapytest.org")mw = HttpCompressionMiddleware.from_crawler(crawler)mw.open_spider(spider)- response = self._getresponse(f"bomb-{compression_id}")- with pytest.raises(IgnoreRequest):+ response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000def test_compression_bomb_setting_br(self):_skip_if_no_br()@@ -539,7 +534,7 @@ class TestHttpCompression:def _test_compression_bomb_spider_attr(self, compression_id):class DownloadMaxSizeSpider(Spider):- download_maxsize = 10_000_000+ download_maxsize = 1_000_000crawler = get_crawler(DownloadMaxSizeSpider)spider = crawler._create_spider("scrapytest.org")@@ -547,8 +542,9 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- with pytest.raises(IgnoreRequest):+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_br(self):@@ -577,9 +573,10 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- response.meta["download_maxsize"] = 10_000_000- with pytest.raises(IgnoreRequest):+ response.meta["download_maxsize"] = 1_000_000+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000def test_compression_bomb_request_meta_br(self):_skip_if_no_br()@@ -728,3 +725,34 @@ class TestHttpCompression:_skip_if_no_zstd()self._test_download_warnsize_request_meta("zstd")++ def _get_truncated_response(self, compression_id):+ crawler = get_crawler(Spider)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)+ response = self._getresponse(compression_id)+ truncated_body = response.body[: len(response.body) // 2]+ response = response.replace(body=truncated_body)+ return mw.process_response(response.request, response)++ def test_process_truncated_response_br(self):+ _skip_if_no_br()+ resp = self._get_truncated_response("br")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_zlibdeflate(self):+ resp = self._get_truncated_response("zlibdeflate")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_gzip(self):+ resp = self._get_truncated_response("gzip")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_zstd(self):+ _skip_if_no_zstd()+ for check_key in FORMAT:+ if not check_key.startswith("zstd-"):+ continue+ resp = self._get_truncated_response(check_key)+ assert len(resp.body) == 0diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 61d79743d..27a78f802 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -61,6 +61,7 @@ def _wrong_credentials(proxy_url):return urlunsplit(bad_auth_proxy)[email protected]_mitmproxyclass TestProxyConnect:@classmethoddef setup_class(cls):@@ -72,13 +73,7 @@ class TestProxyConnect:cls.mockserver.__exit__(None, None, None)def setup_method(self):- try:- import mitmproxy # noqa: F401,PLC0415- except ImportError:- pytest.skip("mitmproxy is not installed")-self._oldenv = os.environ.copy()-self._proxy = MitmProxy()proxy_url = self._proxy.start()os.environ["https_proxy"] = proxy_urldiff --git a/tox.ini b/tox.iniindex c7a41f520..ad5f17595 100644--- a/tox.ini+++ b/tox.ini@@ -25,9 +25,6 @@ deps =deps ={[test-requirements]deps}pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502-- # mitmproxy does not support PyPy- mitmproxy; implementation_name != "pypy"passenv =S3_TEST_FILE_URIAWS_ACCESS_KEY_ID@@ -112,9 +109,6 @@ deps =w3lib==1.17.0zope.interface==5.1.0{[test-requirements]deps}-- # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies- # above, hence we do not install it in pinned environments at the momentsetenv =_SCRAPY_PINNED=truecommands =@@ -137,8 +131,7 @@ deps =Twisted[http2]boto3bpython # optional for shell wrapper tests- brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests- brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests+ brotli >= 1.2.0 # optional for HTTP compress downloader middleware testsgoogle-cloud-storageipythonrobotexclusionrulesparser@@ -152,9 +145,7 @@ deps =Pillow==8.3.2boto3==1.20.0bpython==0.7.1- brotli==0.5.2; implementation_name != "pypy"- brotlicffi==0.8.0; implementation_name == "pypy"- brotlipy+ brotli==1.2.0google-cloud-storage==1.29.0ipython==7.1.0robotexclusionrulesparser==1.6.2@@ -254,7 +245,7 @@ deps ={[testenv]deps}botocore>=1.13.45commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore[testenv:botocore-pinned]basepython = {[pinned]basepython}@@ -264,4 +255,17 @@ deps =setenv ={[pinned]setenv}commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore+++# Run proxy tests that use mitmproxy in a separate env to avoid installing+# numerous mitmproxy deps in other envs (even in extra-deps), as they can+# conflict with other deps we want, or don't want, to have installed there.++[testenv:mitmproxy]+deps =+ {[testenv]deps}+ # mitmproxy does not support PyPy+ mitmproxy; implementation_name != "pypy"+commands =+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy