Scrapy, a fast high-level web crawling & scraping framework for Python.
WIP (#8153)
Adrian7ef4058aa753
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 52f3f4ff1..5ab9eac12 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -22,6 +22,7 @@ from scrapy.utils._compression import (_unzstd,gunzip,)+from scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.deprecate import warn_on_deprecated_spider_attribute@@ -34,6 +35,13 @@ logger = getLogger(__name__)ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br", b"zstd"]+# Dispatching to a thread costs a few hundred microseconds. That is+# negligible next to network latency, but wasted when there is no other+# in-flight request for the reactor to get back to; above this compressed+# size, decoding concurrently with other requests outweighs that cost.+# Tuned by tests/benchmarks/test_httpcompression.py.+_THREAD_DECODE_MIN_SIZE = 8192+class HttpCompressionMiddleware:"""This middleware allows compressed (gzip, deflate etc.) traffic to be@@ -85,7 +93,7 @@ class HttpCompressionMiddleware:return None@_warn_spider_arg- def process_response(+ async def process_response(self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if request.method == "HEAD":@@ -95,7 +103,7 @@ class HttpCompressionMiddleware:max_size = request.meta.get("download_maxsize", self._max_size)warn_size = request.meta.get("download_warnsize", self._warn_size)try:- decoded_body, content_encoding = self._handle_encoding(+ decoded_body, content_encoding = await self._handle_encoding(response.body, content_encoding, max_size)except _DecompressionMaxSizeExceeded as e:@@ -140,12 +148,15 @@ class HttpCompressionMiddleware:del response.headers["Content-Encoding"]return response- def _handle_encoding(+ async def _handle_encoding(self, body: bytes, content_encoding: list[bytes], max_size: int) -> tuple[bytes, list[bytes]]:to_decode, to_keep = self._split_encodings(content_encoding)for encoding in to_decode:- body = self._decode(body, encoding, max_size)+ if len(body) >= _THREAD_DECODE_MIN_SIZE:+ body = await run_in_thread(self._decode, body, encoding, max_size)+ else:+ body = self._decode(body, encoding, max_size)return body, to_keep@staticmethoddiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 96194ee86..965cec7d4 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,3 +1,4 @@+import asyncioimport zlibfrom gzip import GzipFilefrom io import BytesIO@@ -26,6 +27,7 @@ from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceededfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests import tests_datadir+from tests.utils.decorators import coroutine_testSAMPLEDIR = Path(tests_datadir, "compressed")@@ -113,7 +115,8 @@ class TestHttpCompression:HttpCompressionMiddleware,)- def test_no_crawler_constructor(self):+ @coroutine_test+ async def test_no_crawler_constructor(self):with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"):mw = HttpCompressionMiddleware()buf = BytesIO()@@ -126,7 +129,7 @@ class TestHttpCompression:body=body,headers={"Content-Encoding": "gzip"},)- newresponse = mw.process_response(request, response)+ newresponse = await mw.process_response(request, response)assert newresponse.body == b"hello"def test_process_request(self):@@ -135,32 +138,35 @@ class TestHttpCompression:self.mw.process_request(request)assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)- def test_process_response_gzip(self):+ @coroutine_test+ async def test_process_response_gzip(self):response = self._getresponse("gzip")assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"gzip"- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)- def test_process_response_br(self):+ @coroutine_test+ async def test_process_response_br(self):response = self._getresponse("br")assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"br"- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)- def test_process_response_zstd(self):+ @coroutine_test+ async def test_process_response_zstd(self):raw_content = Nonefor check_key in FORMAT:if not check_key.startswith("zstd-"):@@ -169,7 +175,7 @@ class TestHttpCompression:assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"zstd"- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)if raw_content is None:raw_content = newresponse.bodyelse:@@ -178,62 +184,68 @@ class TestHttpCompression:assert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers- def test_process_response_rawdeflate(self):+ @coroutine_test+ async def test_process_response_rawdeflate(self):response = self._getresponse("rawdeflate")assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"deflate"- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74840)- def test_process_response_zlibdelate(self):+ @coroutine_test+ async def test_process_response_zlibdelate(self):response = self._getresponse("zlibdeflate")assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"deflate"- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74840)- def test_process_response_plain(self):+ @coroutine_test+ async def test_process_response_plain(self):response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")request = Request("http://scrapytest.org")assert not response.headers.get("Content-Encoding")- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is responseassert newresponse.body.startswith(b"<!DOCTYPE")self.assertStatsEqual("httpcompression/response_count", None)self.assertStatsEqual("httpcompression/response_bytes", None)- def test_multipleencodings(self):+ @coroutine_test+ async def test_multipleencodings(self):response = self._getresponse("gzip")response.headers["Content-Encoding"] = ["uuencode", "gzip"]assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]- def test_multi_compression_single_header(self):+ @coroutine_test+ async def test_multi_compression_single_header(self):response = self._getresponse("gzip-deflate")assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")- def test_multi_compression_single_header_invalid_compression(+ @coroutine_test+ async def test_multi_compression_single_header_invalid_compression(self, caplog: pytest.LogCaptureFixture) -> None:response = self._getresponse("gzip-deflate")@@ -244,7 +256,7 @@ class TestHttpCompression:with caplog.at_level(WARNING, logger="scrapy.downloadermiddlewares.httpcompression"):- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert caplog.record_tuples == [("scrapy.downloadermiddlewares.httpcompression",@@ -258,45 +270,52 @@ class TestHttpCompression:assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]- def test_multi_compression_multiple_header(self):+ @coroutine_test+ async def test_multi_compression_multiple_header(self):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "deflate"]assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")- def test_multi_compression_multiple_header_invalid_compression(self):+ @coroutine_test+ async def test_multi_compression_multiple_header_invalid_compression(self):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]- def test_multi_compression_single_and_multiple_header(self):+ @coroutine_test+ async def test_multi_compression_single_and_multiple_header(self):response = self._getresponse("gzip-deflate-gzip")response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")- def test_multi_compression_single_and_multiple_header_invalid_compression(self):+ @coroutine_test+ async def test_multi_compression_single_and_multiple_header_invalid_compression(+ self,+ ):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]- def test_process_response_encoding_inside_body(self):+ @coroutine_test+ async def test_process_response_encoding_inside_body(self):headers = {"Content-Type": "text/html","Content-Encoding": "gzip",@@ -314,14 +333,15 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")self.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))- def test_process_response_force_recalculate_encoding(self):+ @coroutine_test+ async def test_process_response_force_recalculate_encoding(self):headers = {"Content-Type": "text/html","Content-Encoding": "gzip",@@ -339,14 +359,15 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")self.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))- def test_process_response_no_content_type_header(self):+ @coroutine_test+ async def test_process_response_no_content_type_header(self):headers = {b"Content-Encoding": b"identity"}plainbody = (b"<html><head><title>Some page</title>"@@ -360,7 +381,7 @@ class TestHttpCompression:)request = Request("http://www.example.com/index")- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert isinstance(newresponse, respcls)assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbody@@ -368,46 +389,50 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))- def test_process_response_gzipped_contenttype(self):+ @coroutine_test+ async def test_process_response_gzipped_contenttype(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/gzip"assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)- def test_process_response_gzip_app_octetstream_contenttype(self):+ @coroutine_test+ async def test_process_response_gzip_app_octetstream_contenttype(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/octet-stream"assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)- def test_process_response_gzip_binary_octetstream_contenttype(self):+ @coroutine_test+ async def test_process_response_gzip_binary_octetstream_contenttype(self):response = self._getresponse("x-gzip")response.headers["Content-Type"] = "binary/octet-stream"assert response.requestrequest = response.request- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headersself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)- def test_process_response_gzipped_gzip_file(self):+ @coroutine_test+ async def test_process_response_gzipped_gzip_file(self):"""Test that a gzip Content-Encoded .gz file is gunzippedonly once by the middleware, leaving gunzipping of the fileto upper layers.@@ -448,19 +473,20 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert gunzip(newresponse.body) == plainbodyself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", len(f.getvalue()))- def test_process_response_head_request_no_decode_required(self):+ @coroutine_test+ async def test_process_response_head_request_no_decode_required(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/gzip"assert response.requestrequest = response.requestrequest.method = "HEAD"response = response.replace(body=None)- newresponse = self.mw.process_response(request, response)+ newresponse = await self.mw.process_response(request, response)assert newresponse is responseassert response.body == b""self.assertStatsEqual("httpcompression/response_count", None)@@ -491,7 +517,9 @@ class TestHttpCompression:result: list[Request | Response] = []thread = Thread(target=lambda: result.append(- self.mw.process_response(Request("http://example.com"), response)+ asyncio.run(+ self.mw.process_response(Request("http://example.com"), response)+ )),daemon=True,)@@ -517,7 +545,7 @@ class TestHttpCompression:)assert decompressor.flush() == b"", f"pending output for size {size}"- def _test_compression_bomb_setting(self, compression_id: str) -> None:+ async def _test_compression_bomb_setting(self, compression_id: str) -> None:settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)spider = crawler._create_spider("scrapytest.org")@@ -527,7 +555,7 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 Bassert response.requestwith pytest.raises(IgnoreRequest) as exc_info:- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)cause = exc_info.value.__cause__assert isinstance(cause, _DecompressionMaxSizeExceeded)assert cause.decompressed_size < 1_100_000@@ -536,19 +564,24 @@ class TestHttpCompression:"exceeded the specified maximum (1000000 B).")- def test_compression_bomb_setting_br(self):- self._test_compression_bomb_setting("br")+ @coroutine_test+ async def test_compression_bomb_setting_br(self):+ await self._test_compression_bomb_setting("br")- def test_compression_bomb_setting_deflate(self):- self._test_compression_bomb_setting("deflate")+ @coroutine_test+ async def test_compression_bomb_setting_deflate(self):+ await self._test_compression_bomb_setting("deflate")- def test_compression_bomb_setting_gzip(self):- self._test_compression_bomb_setting("gzip")+ @coroutine_test+ async def test_compression_bomb_setting_gzip(self):+ await self._test_compression_bomb_setting("gzip")- def test_compression_bomb_setting_zstd(self):- self._test_compression_bomb_setting("zstd")+ @coroutine_test+ async def test_compression_bomb_setting_zstd(self):+ await self._test_compression_bomb_setting("zstd")- def test_compression_bomb_setting_logs_warning(self, caplog):+ @coroutine_test+ async def test_compression_bomb_setting_logs_warning(self, caplog):settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)spider = crawler._create_spider("scrapytest.org")@@ -564,7 +597,7 @@ class TestHttpCompression:),pytest.raises(IgnoreRequest) as exc_info,):- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)assert caplog.record_tuples == [("scrapy.downloadermiddlewares.httpcompression",@@ -573,7 +606,7 @@ class TestHttpCompression:)]- def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:+ async def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:class DownloadMaxSizeSpider(Spider):download_maxsize = 1_000_000@@ -585,28 +618,32 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}")assert response.requestwith pytest.raises(IgnoreRequest) as exc_info:- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)cause = exc_info.value.__cause__assert isinstance(cause, _DecompressionMaxSizeExceeded)assert cause.decompressed_size < 1_100_000@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_compression_bomb_spider_attr_br(self):- self._test_compression_bomb_spider_attr("br")+ @coroutine_test+ async def test_compression_bomb_spider_attr_br(self):+ await self._test_compression_bomb_spider_attr("br")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_compression_bomb_spider_attr_deflate(self):- self._test_compression_bomb_spider_attr("deflate")+ @coroutine_test+ async def test_compression_bomb_spider_attr_deflate(self):+ await self._test_compression_bomb_spider_attr("deflate")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_compression_bomb_spider_attr_gzip(self):- self._test_compression_bomb_spider_attr("gzip")+ @coroutine_test+ async def test_compression_bomb_spider_attr_gzip(self):+ await self._test_compression_bomb_spider_attr("gzip")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_compression_bomb_spider_attr_zstd(self):- self._test_compression_bomb_spider_attr("zstd")+ @coroutine_test+ async def test_compression_bomb_spider_attr_zstd(self):+ await self._test_compression_bomb_spider_attr("zstd")- def _test_compression_bomb_request_meta(self, compression_id: str) -> None:+ async def _test_compression_bomb_request_meta(self, compression_id: str) -> None:crawler = get_crawler(Spider)spider = crawler._create_spider("scrapytest.org")mw = build_from_crawler(HttpCompressionMiddleware, crawler)@@ -616,24 +653,28 @@ class TestHttpCompression:response.meta["download_maxsize"] = 1_000_000assert response.requestwith pytest.raises(IgnoreRequest) as exc_info:- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)cause = exc_info.value.__cause__assert isinstance(cause, _DecompressionMaxSizeExceeded)assert cause.decompressed_size < 1_100_000- def test_compression_bomb_request_meta_br(self):- self._test_compression_bomb_request_meta("br")+ @coroutine_test+ async def test_compression_bomb_request_meta_br(self):+ await self._test_compression_bomb_request_meta("br")- def test_compression_bomb_request_meta_deflate(self):- self._test_compression_bomb_request_meta("deflate")+ @coroutine_test+ async def test_compression_bomb_request_meta_deflate(self):+ await self._test_compression_bomb_request_meta("deflate")- def test_compression_bomb_request_meta_gzip(self):- self._test_compression_bomb_request_meta("gzip")+ @coroutine_test+ async def test_compression_bomb_request_meta_gzip(self):+ await self._test_compression_bomb_request_meta("gzip")- def test_compression_bomb_request_meta_zstd(self):- self._test_compression_bomb_request_meta("zstd")+ @coroutine_test+ async def test_compression_bomb_request_meta_zstd(self):+ await self._test_compression_bomb_request_meta("zstd")- def _test_download_warnsize_setting(+ async def _test_download_warnsize_setting(self, caplog: pytest.LogCaptureFixture, compression_id: str) -> None:settings = {"DOWNLOAD_WARNSIZE": 10_000_000}@@ -648,7 +689,7 @@ class TestHttpCompression:with caplog.at_level(WARNING, logger="scrapy.downloadermiddlewares.httpcompression"):- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)assert caplog.record_tuples == [("scrapy.downloadermiddlewares.httpcompression",@@ -661,27 +702,31 @@ class TestHttpCompression:),]- def test_download_warnsize_setting_br(+ @coroutine_test+ async def test_download_warnsize_setting_br(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_setting(caplog, "br")+ await self._test_download_warnsize_setting(caplog, "br")- def test_download_warnsize_setting_deflate(+ @coroutine_test+ async def test_download_warnsize_setting_deflate(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_setting(caplog, "deflate")+ await self._test_download_warnsize_setting(caplog, "deflate")- def test_download_warnsize_setting_gzip(+ @coroutine_test+ async def test_download_warnsize_setting_gzip(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_setting(caplog, "gzip")+ await self._test_download_warnsize_setting(caplog, "gzip")- def test_download_warnsize_setting_zstd(+ @coroutine_test+ async def test_download_warnsize_setting_zstd(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_setting(caplog, "zstd")+ await self._test_download_warnsize_setting(caplog, "zstd")- def _test_download_warnsize_spider_attr(+ async def _test_download_warnsize_spider_attr(self, caplog: pytest.LogCaptureFixture, compression_id: str) -> None:class DownloadWarnSizeSpider(Spider):@@ -698,7 +743,7 @@ class TestHttpCompression:with caplog.at_level(WARNING, logger="scrapy.downloadermiddlewares.httpcompression"):- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)assert caplog.record_tuples == [("scrapy.downloadermiddlewares.httpcompression",@@ -712,30 +757,34 @@ class TestHttpCompression:]@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_download_warnsize_spider_attr_br(+ @coroutine_test+ async def test_download_warnsize_spider_attr_br(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_spider_attr(caplog, "br")+ await self._test_download_warnsize_spider_attr(caplog, "br")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_download_warnsize_spider_attr_deflate(+ @coroutine_test+ async def test_download_warnsize_spider_attr_deflate(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_spider_attr(caplog, "deflate")+ await self._test_download_warnsize_spider_attr(caplog, "deflate")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_download_warnsize_spider_attr_gzip(+ @coroutine_test+ async def test_download_warnsize_spider_attr_gzip(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_spider_attr(caplog, "gzip")+ await self._test_download_warnsize_spider_attr(caplog, "gzip")@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_download_warnsize_spider_attr_zstd(+ @coroutine_test+ async def test_download_warnsize_spider_attr_zstd(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_spider_attr(caplog, "zstd")+ await self._test_download_warnsize_spider_attr(caplog, "zstd")- def _test_download_warnsize_request_meta(+ async def _test_download_warnsize_request_meta(self, caplog: pytest.LogCaptureFixture, compression_id: str) -> None:crawler = get_crawler(Spider)@@ -750,7 +799,7 @@ class TestHttpCompression:with caplog.at_level(WARNING, logger="scrapy.downloadermiddlewares.httpcompression"):- mw.process_response(response.request, response)+ await mw.process_response(response.request, response)assert caplog.record_tuples == [("scrapy.downloadermiddlewares.httpcompression",@@ -763,27 +812,31 @@ class TestHttpCompression:),]- def test_download_warnsize_request_meta_br(+ @coroutine_test+ async def test_download_warnsize_request_meta_br(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_request_meta(caplog, "br")+ await self._test_download_warnsize_request_meta(caplog, "br")- def test_download_warnsize_request_meta_deflate(+ @coroutine_test+ async def test_download_warnsize_request_meta_deflate(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_request_meta(caplog, "deflate")+ await self._test_download_warnsize_request_meta(caplog, "deflate")- def test_download_warnsize_request_meta_gzip(+ @coroutine_test+ async def test_download_warnsize_request_meta_gzip(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_request_meta(caplog, "gzip")+ await self._test_download_warnsize_request_meta(caplog, "gzip")- def test_download_warnsize_request_meta_zstd(+ @coroutine_test+ async def test_download_warnsize_request_meta_zstd(self, caplog: pytest.LogCaptureFixture) -> None:- self._test_download_warnsize_request_meta(caplog, "zstd")+ await self._test_download_warnsize_request_meta(caplog, "zstd")- def _get_truncated_response(self, compression_id: str) -> Response:+ async def _get_truncated_response(self, compression_id: str) -> Response:crawler = get_crawler(Spider)spider = crawler._create_spider("scrapytest.org")mw = build_from_crawler(HttpCompressionMiddleware, crawler)@@ -792,27 +845,31 @@ class TestHttpCompression:truncated_body = response.body[: len(response.body) // 2]response = response.replace(body=truncated_body)assert response.request- new_response = mw.process_response(response.request, response)+ new_response = await mw.process_response(response.request, response)assert isinstance(new_response, Response)return new_response- def test_process_truncated_response_br(self):- resp = self._get_truncated_response("br")+ @coroutine_test+ async def test_process_truncated_response_br(self):+ resp = await self._get_truncated_response("br")assert resp.body.startswith(b"<!DOCTYPE")- def test_process_truncated_response_zlibdeflate(self):- resp = self._get_truncated_response("zlibdeflate")+ @coroutine_test+ async def test_process_truncated_response_zlibdeflate(self):+ resp = await self._get_truncated_response("zlibdeflate")assert resp.body.startswith(b"<!DOCTYPE")- def test_process_truncated_response_gzip(self):- resp = self._get_truncated_response("gzip")+ @coroutine_test+ async def test_process_truncated_response_gzip(self):+ resp = await self._get_truncated_response("gzip")assert resp.body.startswith(b"<!DOCTYPE")- def test_process_truncated_response_zstd(self):+ @coroutine_test+ async def test_process_truncated_response_zstd(self):for check_key in FORMAT:if not check_key.startswith("zstd-"):continue- resp = self._get_truncated_response(check_key)+ resp = await self._get_truncated_response(check_key)assert len(resp.body) == 0@pytest.mark.parametrize(