Scrapy, a fast high-level web crawling & scraping framework for Python.
Cherry-pick: Mitigate brotli and deflate decompression bombs DoS (#7134)
Rui Xic44b8df6c7f8
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 06da46ca1..5d13f0c1f 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -68,6 +68,9 @@ jobs:- python-version: "3.13"env:TOXENV: botocore+ - python-version: "3.13"+ env:+ TOXENV: mitmproxysteps:- uses: actions/checkout@v4diff --git a/conftest.py b/conftest.pyindex 8e0c429a0..9ffcff726 100644--- a/conftest.py+++ b/conftest.py@@ -116,6 +116,16 @@ def requires_boto3(request):pytest.skip("boto3 is not installed")[email protected](autouse=True)+def requires_mitmproxy(request):+ if not request.node.get_closest_marker("requires_mitmproxy"):+ return+ try:+ import mitmproxy # noqa: F401+ except ImportError:+ pytest.skip("mitmproxy is not installed")++def pytest_configure(config):if config.getoption("--reactor") != "default":install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")diff --git a/pyproject.toml b/pyproject.tomlindex e1cfec355..f81fb295e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -243,6 +243,7 @@ markers = ["requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)","requires_boto3: marks tests that need botocore and boto3",+ "requires_mitmproxy: marks tests that need mitmproxy",]filterwarnings = ["ignore::DeprecationWarning:twisted.web.static"diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 58891b952..4190a6547 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,5 +1,6 @@from __future__ import annotations+import warningsfrom itertools import chainfrom logging import getLoggerfrom typing import TYPE_CHECKING, Any@@ -29,14 +30,20 @@ logger = getLogger(__name__)ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"]try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401+ import brotliexcept ImportError:passelse:- ACCEPTED_ENCODINGS.append(b"br")+ try:+ brotli.Decompressor.can_accept_more_data+ except AttributeError: # pragma: no cover+ warnings.warn(+ "You have brotli installed. But 'br' encoding support now requires "+ "brotli version >= 1.2.0. Please upgrade brotli version to make Scrapy "+ "decode 'br' encoded responses.",+ )+ else:+ ACCEPTED_ENCODINGS.append(b"br")try:import zstandard # noqa: F401@@ -98,13 +105,13 @@ class HttpCompressionMiddleware:decoded_body, content_encoding = self._handle_encoding(response.body, content_encoding, max_size)- except _DecompressionMaxSizeExceeded:+ except _DecompressionMaxSizeExceeded as e:raise IgnoreRequest(f"Ignored response {response} because its body "- f"({len(response.body)} B compressed) exceeded "- f"DOWNLOAD_MAXSIZE ({max_size} B) during "- f"decompression."- )+ f"({len(response.body)} B compressed, "+ f"{e.decompressed_size} B decompressed so far) exceeded "+ f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."+ ) from eif len(response.body) < warn_size <= len(decoded_body):logger.warning(f"{response} body size after decompression "@@ -187,7 +194,7 @@ class HttpCompressionMiddleware:f"from unsupported encoding(s) '{encodings_str}'.")if b"br" in encodings:- msg += " You need to install brotli or brotlicffi to decode 'br'."+ msg += " You need to install brotli >= 1.2.0 to decode 'br'."if b"zstd" in encodings:msg += " You need to install zstandard to decode 'zstd'."logger.warning(msg)diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 6b09f36ff..0a8191cc6 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -1,42 +1,9 @@import contextlibimport zlibfrom io import BytesIO-from warnings import warn--from scrapy.exceptions import ScrapyDeprecationWarning--try:- try:- import brotli- except ImportError:- import brotlicffi as brotli-except ImportError:- pass-else:- try:- brotli.Decompressor.process- except AttributeError:- warn(- (- "You have brotlipy installed, and Scrapy will use it, but "- "Scrapy support for brotlipy is deprecated and will stop "- "working in a future version of Scrapy. brotlipy itself is "- "deprecated, it has been superseded by brotlicffi. Please, "- "uninstall brotlipy and install brotli or brotlicffi instead. "- "brotlipy has the same import name as brotli, so keeping both "- "installed is strongly discouraged."- ),- ScrapyDeprecationWarning,- )-- def _brotli_decompress(decompressor, data):- return decompressor.decompress(data)-- else:-- def _brotli_decompress(decompressor, data):- return decompressor.process(data)+with contextlib.suppress(ImportError):+ import brotliwith contextlib.suppress(ImportError):import zstandard@@ -46,62 +13,64 @@ _CHUNK_SIZE = 65536 # 64 KiBclass _DecompressionMaxSizeExceeded(ValueError):- pass+ def __init__(self, decompressed_size: int, max_size: int) -> None:+ self.decompressed_size = decompressed_size+ self.max_size = max_size++ def __str__(self) -> str:+ return (+ f"The number of bytes decompressed so far "+ f"({self.decompressed_size} B) exceeded the specified maximum "+ f"({self.max_size} B)."+ )+++def _check_max_size(decompressed_size: int, max_size: int) -> None:+ if max_size and decompressed_size > max_size:+ raise _DecompressionMaxSizeExceeded(decompressed_size, max_size)def _inflate(data: bytes, *, max_size: int = 0) -> bytes:decompressor = zlib.decompressobj()- raw_decompressor = zlib.decompressobj(wbits=-15)- input_stream = BytesIO(data)+ try:+ first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE)+ except zlib.error:+ # to work with raw deflate content that may be sent by microsoft servers.+ decompressor = zlib.decompressobj(wbits=-15)+ first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE)+ decompressed_size = len(first_chunk)+ _check_max_size(decompressed_size, max_size)output_stream = BytesIO()- output_chunk = b"."- decompressed_size = 0- while output_chunk:- input_chunk = input_stream.read(_CHUNK_SIZE)- try:- output_chunk = decompressor.decompress(input_chunk)- except zlib.error:- if decompressor != raw_decompressor:- # ugly hack to work with raw deflate content that may- # be sent by microsoft servers. For more information, see:- # http://carsten.codimi.de/gzip.yaws/- # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx- # http://www.gzip.org/zlib/zlib_faq.html#faq38- decompressor = raw_decompressor- output_chunk = decompressor.decompress(input_chunk)- else:- raise+ output_stream.write(first_chunk)+ while decompressor.unconsumed_tail:+ output_chunk = decompressor.decompress(+ decompressor.unconsumed_tail, max_length=_CHUNK_SIZE+ )decompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ if tail := decompressor.flush():+ decompressed_size += len(tail)+ _check_max_size(decompressed_size, max_size)+ output_stream.write(tail)+ return output_stream.getvalue()def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:decompressor = brotli.Decompressor()- input_stream = BytesIO(data)+ first_chunk = decompressor.process(data, output_buffer_limit=_CHUNK_SIZE)+ decompressed_size = len(first_chunk)+ _check_max_size(decompressed_size, max_size)output_stream = BytesIO()- output_chunk = b"."- decompressed_size = 0- while output_chunk:- input_chunk = input_stream.read(_CHUNK_SIZE)- output_chunk = _brotli_decompress(decompressor, input_chunk)+ output_stream.write(first_chunk)+ while not decompressor.is_finished():+ output_chunk = decompressor.process(b"", output_buffer_limit=_CHUNK_SIZE)+ if not output_chunk:+ breakdecompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:@@ -113,12 +82,6 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:while output_chunk:output_chunk = stream_reader.read(_CHUNK_SIZE)decompressed_size += len(output_chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(output_chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex 85324361c..5f95bccbd 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -5,7 +5,7 @@ from gzip import GzipFilefrom io import BytesIOfrom typing import TYPE_CHECKING-from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded+from ._compression import _CHUNK_SIZE, _check_max_sizeif TYPE_CHECKING:from scrapy.http import Response@@ -31,15 +31,9 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:breakraisedecompressed_size += len(chunk)- if max_size and decompressed_size > max_size:- raise _DecompressionMaxSizeExceeded(- f"The number of bytes decompressed so far "- f"({decompressed_size} B) exceed the specified maximum "- f"({max_size} B)."- )+ _check_max_size(decompressed_size, max_size)output_stream.write(chunk)- output_stream.seek(0)- return output_stream.read()+ return output_stream.getvalue()def gzip_magic_number(response: Response) -> bool:diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex e7427c5ac..7e068587d 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -51,6 +51,22 @@ FORMAT = {}+def _skip_if_no_br() -> None:+ try:+ import brotli++ brotli.Decompressor.can_accept_more_data+ except (ImportError, AttributeError):+ pytest.skip("no brotli support")+++def _skip_if_no_zstd() -> None:+ try:+ import zstandard # noqa: F401+ except ImportError:+ pytest.skip("no zstd support (zstandard)")++class TestHttpCompression:def setup_method(self):self.crawler = get_crawler(Spider)@@ -124,13 +140,7 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", 74837)def test_process_response_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()response = self._getresponse("br")request = response.requestassert response.headers["Content-Encoding"] == b"br"@@ -143,14 +153,9 @@ class TestHttpCompression:def test_process_response_br_unsupported(self):try:- try:- import brotli # noqa: F401+ import brotli # noqa: F401- raise SkipTest("Requires not having brotli support")- except ImportError:- import brotlicffi # noqa: F401-- raise SkipTest("Requires not having brotli support")+ raise SkipTest("Requires not having brotli support")except ImportError:passresponse = self._getresponse("br")@@ -169,7 +174,7 @@ class TestHttpCompression:("HttpCompressionMiddleware cannot decode the response for"" http://scrapytest.org/ from unsupported encoding(s) 'br'."- " You need to install brotli or brotlicffi to decode 'br'."+ " You need to install brotli >= 1.2.0 to decode 'br'."),),)@@ -177,10 +182,7 @@ class TestHttpCompression:assert newresponse.headers.getlist("Content-Encoding") == [b"br"]def test_process_response_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()raw_content = Nonefor check_key in FORMAT:if not check_key.startswith("zstd-"):@@ -503,24 +505,19 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", None)def _test_compression_bomb_setting(self, compression_id):- settings = {"DOWNLOAD_MAXSIZE": 10_000_000}+ settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)spider = crawler._create_spider("scrapytest.org")mw = HttpCompressionMiddleware.from_crawler(crawler)mw.open_spider(spider)- response = self._getresponse(f"bomb-{compression_id}")- with pytest.raises(IgnoreRequest):+ response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response, spider)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000def test_compression_bomb_setting_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_compression_bomb_setting("br")def test_compression_bomb_setting_deflate(self):@@ -530,15 +527,12 @@ class TestHttpCompression:self._test_compression_bomb_setting("gzip")def test_compression_bomb_setting_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_compression_bomb_setting("zstd")def _test_compression_bomb_spider_attr(self, compression_id):class DownloadMaxSizeSpider(Spider):- download_maxsize = 10_000_000+ download_maxsize = 1_000_000crawler = get_crawler(DownloadMaxSizeSpider)spider = crawler._create_spider("scrapytest.org")@@ -546,17 +540,12 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- with pytest.raises(IgnoreRequest):+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response, spider)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000def test_compression_bomb_spider_attr_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_compression_bomb_spider_attr("br")def test_compression_bomb_spider_attr_deflate(self):@@ -566,10 +555,7 @@ class TestHttpCompression:self._test_compression_bomb_spider_attr("gzip")def test_compression_bomb_spider_attr_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_compression_bomb_spider_attr("zstd")def _test_compression_bomb_request_meta(self, compression_id):@@ -579,18 +565,13 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- response.meta["download_maxsize"] = 10_000_000- with pytest.raises(IgnoreRequest):+ response.meta["download_maxsize"] = 1_000_000+ with pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response, spider)+ assert exc_info.value.__cause__.decompressed_size < 1_100_000def test_compression_bomb_request_meta_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_compression_bomb_request_meta("br")def test_compression_bomb_request_meta_deflate(self):@@ -600,10 +581,7 @@ class TestHttpCompression:self._test_compression_bomb_request_meta("gzip")def test_compression_bomb_request_meta_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_compression_bomb_request_meta("zstd")def _test_download_warnsize_setting(self, compression_id):@@ -633,13 +611,7 @@ class TestHttpCompression:)def test_download_warnsize_setting_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_download_warnsize_setting("br")def test_download_warnsize_setting_deflate(self):@@ -649,10 +621,7 @@ class TestHttpCompression:self._test_download_warnsize_setting("gzip")def test_download_warnsize_setting_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_download_warnsize_setting("zstd")def _test_download_warnsize_spider_attr(self, compression_id):@@ -684,13 +653,7 @@ class TestHttpCompression:)def test_download_warnsize_spider_attr_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_download_warnsize_spider_attr("br")def test_download_warnsize_spider_attr_deflate(self):@@ -700,10 +663,7 @@ class TestHttpCompression:self._test_download_warnsize_spider_attr("gzip")def test_download_warnsize_spider_attr_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_download_warnsize_spider_attr("zstd")def _test_download_warnsize_request_meta(self, compression_id):@@ -733,13 +693,7 @@ class TestHttpCompression:)def test_download_warnsize_request_meta_br(self):- try:- try:- import brotli # noqa: F401- except ImportError:- import brotlicffi # noqa: F401- except ImportError:- raise SkipTest("no brotli")+ _skip_if_no_br()self._test_download_warnsize_request_meta("br")def test_download_warnsize_request_meta_deflate(self):@@ -749,8 +703,36 @@ class TestHttpCompression:self._test_download_warnsize_request_meta("gzip")def test_download_warnsize_request_meta_zstd(self):- try:- import zstandard # noqa: F401- except ImportError:- raise SkipTest("no zstd support (zstandard)")+ _skip_if_no_zstd()self._test_download_warnsize_request_meta("zstd")++ def _get_truncated_response(self, compression_id):+ crawler = get_crawler(Spider)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)+ response = self._getresponse(compression_id)+ truncated_body = response.body[: len(response.body) // 2]+ response = response.replace(body=truncated_body)+ return mw.process_response(response.request, response, spider)++ def test_process_truncated_response_br(self):+ _skip_if_no_br()+ resp = self._get_truncated_response("br")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_zlibdeflate(self):+ resp = self._get_truncated_response("zlibdeflate")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_gzip(self):+ resp = self._get_truncated_response("gzip")+ assert resp.body.startswith(b"<!DOCTYPE")++ def test_process_truncated_response_zstd(self):+ _skip_if_no_zstd()+ for check_key in FORMAT:+ if not check_key.startswith("zstd-"):+ continue+ resp = self._get_truncated_response(check_key)+ assert len(resp.body) == 0diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 885b7b7ae..11dabdf9b 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -62,6 +62,7 @@ def _wrong_credentials(proxy_url):return urlunsplit(bad_auth_proxy)[email protected]_mitmproxyclass TestProxyConnect(TestCase):@classmethoddef setUpClass(cls):@@ -73,13 +74,7 @@ class TestProxyConnect(TestCase):cls.mockserver.__exit__(None, None, None)def setUp(self):- try:- import mitmproxy # noqa: F401- except ImportError:- pytest.skip("mitmproxy is not installed")-self._oldenv = os.environ.copy()-self._proxy = MitmProxy()proxy_url = self._proxy.start()os.environ["https_proxy"] = proxy_urldiff --git a/tox.ini b/tox.iniindex 8abb0dcac..98b944cc8 100644--- a/tox.ini+++ b/tox.ini@@ -25,9 +25,6 @@ deps =deps ={[test-requirements]deps}pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502-- # mitmproxy does not support PyPy- mitmproxy; implementation_name != "pypy"passenv =S3_TEST_FILE_URIAWS_ACCESS_KEY_ID@@ -110,9 +107,6 @@ deps =w3lib==1.17.0zope.interface==5.1.0{[test-requirements]deps}-- # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies- # above, hence we do not install it in pinned environments at the momentsetenv =_SCRAPY_PINNED=trueinstall_command =@@ -139,8 +133,7 @@ deps =Twisted[http2]boto3bpython # optional for shell wrapper tests- brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests- brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests+ brotli >= 1.2.0 # optional for HTTP compress downloader middleware testsgoogle-cloud-storageipythonrobotexclusionrulesparser@@ -154,9 +147,7 @@ deps =Pillow==8.0.0boto3==1.20.0bpython==0.7.1- brotli==0.5.2; implementation_name != "pypy"- brotlicffi==0.8.0; implementation_name == "pypy"- brotlipy+ brotli==1.2.0google-cloud-storage==1.29.0ipython==2.0.0robotexclusionrulesparser==1.6.2@@ -256,7 +247,7 @@ deps ={[testenv]deps}botocore>=1.4.87commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore[testenv:botocore-pinned]basepython = {[pinned]basepython}@@ -267,4 +258,17 @@ install_command = {[pinned]install_command}setenv ={[pinned]setenv}commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore+++# Run proxy tests that use mitmproxy in a separate env to avoid installing+# numerous mitmproxy deps in other envs (even in extra-deps), as they can+# conflict with other deps we want, or don't want, to have installed there.++[testenv:mitmproxy]+deps =+ {[testenv]deps}+ # mitmproxy does not support PyPy+ mitmproxy; implementation_name != "pypy"+commands =+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy