Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

WIP (#8153)

Adrian7ef4058aa753
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 52f3f4ff1..5ab9eac12 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -22,6 +22,7 @@ from scrapy.utils._compression import (
_unzstd,
gunzip,
)
+from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
@@ -34,6 +35,13 @@ logger = getLogger(__name__)
ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br", b"zstd"]
+# Dispatching to a thread costs a few hundred microseconds. That is
+# negligible next to network latency, but wasted when there is no other
+# in-flight request for the reactor to get back to; above this compressed
+# size, decoding concurrently with other requests outweighs that cost.
+# Tuned by tests/benchmarks/test_httpcompression.py.
+_THREAD_DECODE_MIN_SIZE = 8192
+
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
@@ -85,7 +93,7 @@ class HttpCompressionMiddleware:
return None
@_warn_spider_arg
- def process_response(
+ async def process_response(
self, request: Request, response: Response, spider: Spider | None = None
) -> Request | Response:
if request.method == "HEAD":
@@ -95,7 +103,7 @@ class HttpCompressionMiddleware:
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
- decoded_body, content_encoding = self._handle_encoding(
+ decoded_body, content_encoding = await self._handle_encoding(
response.body, content_encoding, max_size
)
except _DecompressionMaxSizeExceeded as e:
@@ -140,12 +148,15 @@ class HttpCompressionMiddleware:
del response.headers["Content-Encoding"]
return response
- def _handle_encoding(
+ async def _handle_encoding(
self, body: bytes, content_encoding: list[bytes], max_size: int
) -> tuple[bytes, list[bytes]]:
to_decode, to_keep = self._split_encodings(content_encoding)
for encoding in to_decode:
- body = self._decode(body, encoding, max_size)
+ if len(body) >= _THREAD_DECODE_MIN_SIZE:
+ body = await run_in_thread(self._decode, body, encoding, max_size)
+ else:
+ body = self._decode(body, encoding, max_size)
return body, to_keep
@staticmethod
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 96194ee86..965cec7d4 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,3 +1,4 @@
+import asyncio
import zlib
from gzip import GzipFile
from io import BytesIO
@@ -26,6 +27,7 @@ from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import tests_datadir
+from tests.utils.decorators import coroutine_test
SAMPLEDIR = Path(tests_datadir, "compressed")
@@ -113,7 +115,8 @@ class TestHttpCompression:
HttpCompressionMiddleware,
)
- def test_no_crawler_constructor(self):
+ @coroutine_test
+ async def test_no_crawler_constructor(self):
with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"):
mw = HttpCompressionMiddleware()
buf = BytesIO()
@@ -126,7 +129,7 @@ class TestHttpCompression:
body=body,
headers={"Content-Encoding": "gzip"},
)
- newresponse = mw.process_response(request, response)
+ newresponse = await mw.process_response(request, response)
assert newresponse.body == b"hello"
def test_process_request(self):
@@ -135,32 +138,35 @@ class TestHttpCompression:
self.mw.process_request(request)
assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)
- def test_process_response_gzip(self):
+ @coroutine_test
+ async def test_process_response_gzip(self):
response = self._getresponse("gzip")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
- def test_process_response_br(self):
+ @coroutine_test
+ async def test_process_response_br(self):
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
- def test_process_response_zstd(self):
+ @coroutine_test
+ async def test_process_response_zstd(self):
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
@@ -169,7 +175,7 @@ class TestHttpCompression:
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
if raw_content is None:
raw_content = newresponse.body
else:
@@ -178,62 +184,68 @@ class TestHttpCompression:
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
- def test_process_response_rawdeflate(self):
+ @coroutine_test
+ async def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
- def test_process_response_zlibdelate(self):
+ @coroutine_test
+ async def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
- def test_process_response_plain(self):
+ @coroutine_test
+ async def test_process_response_plain(self):
response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
request = Request("http://scrapytest.org")
assert not response.headers.get("Content-Encoding")
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is response
assert newresponse.body.startswith(b"<!DOCTYPE")
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
- def test_multipleencodings(self):
+ @coroutine_test
+ async def test_multipleencodings(self):
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]
- def test_multi_compression_single_header(self):
+ @coroutine_test
+ async def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
- def test_multi_compression_single_header_invalid_compression(
+ @coroutine_test
+ async def test_multi_compression_single_header_invalid_compression(
self, caplog: pytest.LogCaptureFixture
) -> None:
response = self._getresponse("gzip-deflate")
@@ -244,7 +256,7 @@ class TestHttpCompression:
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
@@ -258,45 +270,52 @@ class TestHttpCompression:
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
- def test_multi_compression_multiple_header(self):
+ @coroutine_test
+ async def test_multi_compression_multiple_header(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
- def test_multi_compression_multiple_header_invalid_compression(self):
+ @coroutine_test
+ async def test_multi_compression_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
- def test_multi_compression_single_and_multiple_header(self):
+ @coroutine_test
+ async def test_multi_compression_single_and_multiple_header(self):
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
- def test_multi_compression_single_and_multiple_header_invalid_compression(self):
+ @coroutine_test
+ async def test_multi_compression_single_and_multiple_header_invalid_compression(
+ self,
+ ):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
- def test_process_response_encoding_inside_body(self):
+ @coroutine_test
+ async def test_process_response_encoding_inside_body(self):
headers = {
"Content-Type": "text/html",
"Content-Encoding": "gzip",
@@ -314,14 +333,15 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
- def test_process_response_force_recalculate_encoding(self):
+ @coroutine_test
+ async def test_process_response_force_recalculate_encoding(self):
headers = {
"Content-Type": "text/html",
"Content-Encoding": "gzip",
@@ -339,14 +359,15 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
- def test_process_response_no_content_type_header(self):
+ @coroutine_test
+ async def test_process_response_no_content_type_header(self):
headers = {b"Content-Encoding": b"identity"}
plainbody = (
b"<html><head><title>Some page</title>"
@@ -360,7 +381,7 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/index")
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
@@ -368,46 +389,50 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
- def test_process_response_gzipped_contenttype(self):
+ @coroutine_test
+ async def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
- def test_process_response_gzip_app_octetstream_contenttype(self):
+ @coroutine_test
+ async def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/octet-stream"
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
- def test_process_response_gzip_binary_octetstream_contenttype(self):
+ @coroutine_test
+ async def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
response.headers["Content-Type"] = "binary/octet-stream"
assert response.request
request = response.request
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
- def test_process_response_gzipped_gzip_file(self):
+ @coroutine_test
+ async def test_process_response_gzipped_gzip_file(self):
"""Test that a gzip Content-Encoded .gz file is gunzipped
only once by the middleware, leaving gunzipping of the file
to upper layers.
@@ -448,19 +473,20 @@ class TestHttpCompression:
)
request = Request("http://www.example.com/")
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert gunzip(newresponse.body) == plainbody
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(f.getvalue()))
- def test_process_response_head_request_no_decode_required(self):
+ @coroutine_test
+ async def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
- newresponse = self.mw.process_response(request, response)
+ newresponse = await self.mw.process_response(request, response)
assert newresponse is response
assert response.body == b""
self.assertStatsEqual("httpcompression/response_count", None)
@@ -491,7 +517,9 @@ class TestHttpCompression:
result: list[Request | Response] = []
thread = Thread(
target=lambda: result.append(
- self.mw.process_response(Request("http://example.com"), response)
+ asyncio.run(
+ self.mw.process_response(Request("http://example.com"), response)
+ )
),
daemon=True,
)
@@ -517,7 +545,7 @@ class TestHttpCompression:
)
assert decompressor.flush() == b"", f"pending output for size {size}"
- def _test_compression_bomb_setting(self, compression_id: str) -> None:
+ async def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
@@ -527,7 +555,7 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@@ -536,19 +564,24 @@ class TestHttpCompression:
"exceeded the specified maximum (1000000 B)."
)
- def test_compression_bomb_setting_br(self):
- self._test_compression_bomb_setting("br")
+ @coroutine_test
+ async def test_compression_bomb_setting_br(self):
+ await self._test_compression_bomb_setting("br")
- def test_compression_bomb_setting_deflate(self):
- self._test_compression_bomb_setting("deflate")
+ @coroutine_test
+ async def test_compression_bomb_setting_deflate(self):
+ await self._test_compression_bomb_setting("deflate")
- def test_compression_bomb_setting_gzip(self):
- self._test_compression_bomb_setting("gzip")
+ @coroutine_test
+ async def test_compression_bomb_setting_gzip(self):
+ await self._test_compression_bomb_setting("gzip")
- def test_compression_bomb_setting_zstd(self):
- self._test_compression_bomb_setting("zstd")
+ @coroutine_test
+ async def test_compression_bomb_setting_zstd(self):
+ await self._test_compression_bomb_setting("zstd")
- def test_compression_bomb_setting_logs_warning(self, caplog):
+ @coroutine_test
+ async def test_compression_bomb_setting_logs_warning(self, caplog):
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
@@ -564,7 +597,7 @@ class TestHttpCompression:
),
pytest.raises(IgnoreRequest) as exc_info,
):
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
@@ -573,7 +606,7 @@ class TestHttpCompression:
)
]
- def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
+ async def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000
@@ -585,28 +618,32 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_compression_bomb_spider_attr_br(self):
- self._test_compression_bomb_spider_attr("br")
+ @coroutine_test
+ async def test_compression_bomb_spider_attr_br(self):
+ await self._test_compression_bomb_spider_attr("br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_compression_bomb_spider_attr_deflate(self):
- self._test_compression_bomb_spider_attr("deflate")
+ @coroutine_test
+ async def test_compression_bomb_spider_attr_deflate(self):
+ await self._test_compression_bomb_spider_attr("deflate")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_compression_bomb_spider_attr_gzip(self):
- self._test_compression_bomb_spider_attr("gzip")
+ @coroutine_test
+ async def test_compression_bomb_spider_attr_gzip(self):
+ await self._test_compression_bomb_spider_attr("gzip")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_compression_bomb_spider_attr_zstd(self):
- self._test_compression_bomb_spider_attr("zstd")
+ @coroutine_test
+ async def test_compression_bomb_spider_attr_zstd(self):
+ await self._test_compression_bomb_spider_attr("zstd")
- def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
+ async def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
@@ -616,24 +653,28 @@ class TestHttpCompression:
response.meta["download_maxsize"] = 1_000_000
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
- def test_compression_bomb_request_meta_br(self):
- self._test_compression_bomb_request_meta("br")
+ @coroutine_test
+ async def test_compression_bomb_request_meta_br(self):
+ await self._test_compression_bomb_request_meta("br")
- def test_compression_bomb_request_meta_deflate(self):
- self._test_compression_bomb_request_meta("deflate")
+ @coroutine_test
+ async def test_compression_bomb_request_meta_deflate(self):
+ await self._test_compression_bomb_request_meta("deflate")
- def test_compression_bomb_request_meta_gzip(self):
- self._test_compression_bomb_request_meta("gzip")
+ @coroutine_test
+ async def test_compression_bomb_request_meta_gzip(self):
+ await self._test_compression_bomb_request_meta("gzip")
- def test_compression_bomb_request_meta_zstd(self):
- self._test_compression_bomb_request_meta("zstd")
+ @coroutine_test
+ async def test_compression_bomb_request_meta_zstd(self):
+ await self._test_compression_bomb_request_meta("zstd")
- def _test_download_warnsize_setting(
+ async def _test_download_warnsize_setting(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
@@ -648,7 +689,7 @@ class TestHttpCompression:
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
@@ -661,27 +702,31 @@ class TestHttpCompression:
),
]
- def test_download_warnsize_setting_br(
+ @coroutine_test
+ async def test_download_warnsize_setting_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_setting(caplog, "br")
+ await self._test_download_warnsize_setting(caplog, "br")
- def test_download_warnsize_setting_deflate(
+ @coroutine_test
+ async def test_download_warnsize_setting_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_setting(caplog, "deflate")
+ await self._test_download_warnsize_setting(caplog, "deflate")
- def test_download_warnsize_setting_gzip(
+ @coroutine_test
+ async def test_download_warnsize_setting_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_setting(caplog, "gzip")
+ await self._test_download_warnsize_setting(caplog, "gzip")
- def test_download_warnsize_setting_zstd(
+ @coroutine_test
+ async def test_download_warnsize_setting_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_setting(caplog, "zstd")
+ await self._test_download_warnsize_setting(caplog, "zstd")
- def _test_download_warnsize_spider_attr(
+ async def _test_download_warnsize_spider_attr(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
class DownloadWarnSizeSpider(Spider):
@@ -698,7 +743,7 @@ class TestHttpCompression:
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
@@ -712,30 +757,34 @@ class TestHttpCompression:
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_download_warnsize_spider_attr_br(
+ @coroutine_test
+ async def test_download_warnsize_spider_attr_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_spider_attr(caplog, "br")
+ await self._test_download_warnsize_spider_attr(caplog, "br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_download_warnsize_spider_attr_deflate(
+ @coroutine_test
+ async def test_download_warnsize_spider_attr_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_spider_attr(caplog, "deflate")
+ await self._test_download_warnsize_spider_attr(caplog, "deflate")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_download_warnsize_spider_attr_gzip(
+ @coroutine_test
+ async def test_download_warnsize_spider_attr_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_spider_attr(caplog, "gzip")
+ await self._test_download_warnsize_spider_attr(caplog, "gzip")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_download_warnsize_spider_attr_zstd(
+ @coroutine_test
+ async def test_download_warnsize_spider_attr_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_spider_attr(caplog, "zstd")
+ await self._test_download_warnsize_spider_attr(caplog, "zstd")
- def _test_download_warnsize_request_meta(
+ async def _test_download_warnsize_request_meta(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
crawler = get_crawler(Spider)
@@ -750,7 +799,7 @@ class TestHttpCompression:
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
- mw.process_response(response.request, response)
+ await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
@@ -763,27 +812,31 @@ class TestHttpCompression:
),
]
- def test_download_warnsize_request_meta_br(
+ @coroutine_test
+ async def test_download_warnsize_request_meta_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_request_meta(caplog, "br")
+ await self._test_download_warnsize_request_meta(caplog, "br")
- def test_download_warnsize_request_meta_deflate(
+ @coroutine_test
+ async def test_download_warnsize_request_meta_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_request_meta(caplog, "deflate")
+ await self._test_download_warnsize_request_meta(caplog, "deflate")
- def test_download_warnsize_request_meta_gzip(
+ @coroutine_test
+ async def test_download_warnsize_request_meta_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_request_meta(caplog, "gzip")
+ await self._test_download_warnsize_request_meta(caplog, "gzip")
- def test_download_warnsize_request_meta_zstd(
+ @coroutine_test
+ async def test_download_warnsize_request_meta_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
- self._test_download_warnsize_request_meta(caplog, "zstd")
+ await self._test_download_warnsize_request_meta(caplog, "zstd")
- def _get_truncated_response(self, compression_id: str) -> Response:
+ async def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
@@ -792,27 +845,31 @@ class TestHttpCompression:
truncated_body = response.body[: len(response.body) // 2]
response = response.replace(body=truncated_body)
assert response.request
- new_response = mw.process_response(response.request, response)
+ new_response = await mw.process_response(response.request, response)
assert isinstance(new_response, Response)
return new_response
- def test_process_truncated_response_br(self):
- resp = self._get_truncated_response("br")
+ @coroutine_test
+ async def test_process_truncated_response_br(self):
+ resp = await self._get_truncated_response("br")
assert resp.body.startswith(b"<!DOCTYPE")
- def test_process_truncated_response_zlibdeflate(self):
- resp = self._get_truncated_response("zlibdeflate")
+ @coroutine_test
+ async def test_process_truncated_response_zlibdeflate(self):
+ resp = await self._get_truncated_response("zlibdeflate")
assert resp.body.startswith(b"<!DOCTYPE")
- def test_process_truncated_response_gzip(self):
- resp = self._get_truncated_response("gzip")
+ @coroutine_test
+ async def test_process_truncated_response_gzip(self):
+ resp = await self._get_truncated_response("gzip")
assert resp.body.startswith(b"<!DOCTYPE")
- def test_process_truncated_response_zstd(self):
+ @coroutine_test
+ async def test_process_truncated_response_zstd(self):
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
continue
- resp = self._get_truncated_response(check_key)
+ resp = await self._get_truncated_response(check_key)
assert len(resp.body) == 0
@pytest.mark.parametrize(