Scrapy, a fast high-level web crawling & scraping framework for Python.
import asyncio
import zlib
from gzip import GzipFile
from io import BytesIO
from logging import WARNING
from pathlib import Path
from threading import Thread
from typing import Any
import pytest
from w3lib.encoding import resolve_encoding
from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
from scrapy.exceptions import (
DecompressionError,
IgnoreRequest,
NotConfigured,
ScrapyDeprecationWarning,
)
from scrapy.http import BodyFileResponse, HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded, gunzip
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from tests.utils.decorators import coroutine_test
SAMPLEDIR = Path(tests_datadir, "compressed")
FORMAT = {
"gzip": ("html-gzip.bin", "gzip"),
"x-gzip": ("html-gzip.bin", "x-gzip"),
"rawdeflate": ("html-rawdeflate.bin", "deflate"),
"zlibdeflate": ("html-zlibdeflate.bin", "deflate"),
"gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"),
"gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"),
"br": ("html-br.bin", "br"),
# $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
"zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"),
# $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
"zstd-static-no-content-size": ("html-zstd-static-no-content-size.bin", "zstd"),
# $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
"zstd-streaming-no-content-size": (
"html-zstd-streaming-no-content-size.bin",
"zstd",
),
**{
f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
for format_id in (
"br", # 34 → 11 511 612
"deflate", # 27 968 → 11 511 612
"gzip", # 27 988 → 11 511 612
"zstd", # 1 096 → 11 511 612
)
},
}
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)
self.crawler.stats.open_spider()
def _mw(self, max_encodings: int) -> HttpCompressionMiddleware:
crawler = get_crawler(
Spider, settings_dict={"COMPRESSION_MAX_ENCODINGS": max_encodings}
)
return build_from_crawler(HttpCompressionMiddleware, crawler)
def _getresponse(self, coding: str) -> Response:
if coding not in FORMAT:
raise ValueError
samplefile, contentencoding = FORMAT[coding]
body = (SAMPLEDIR / samplefile).read_bytes()
headers = {
"Server": "Yaws/1.49 Yet Another Web Server",
"Date": "Sun, 08 Mar 2009 00:41:03 GMT",
"Content-Length": len(body),
"Content-Type": "text/html",
"Content-Encoding": contentencoding,
}
response = Response("http://scrapytest.org/", body=body, headers=headers)
response.request = Request(
"http://scrapytest.org", headers={"Accept-Encoding": "gzip, deflate"}
)
return response
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
def test_setting_false_compression_enabled(self):
with pytest.raises(NotConfigured):
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
)
def test_setting_default_compression_enabled(self):
assert isinstance(
build_from_crawler(HttpCompressionMiddleware, get_crawler()),
HttpCompressionMiddleware,
)
def test_setting_true_compression_enabled(self):
assert isinstance(
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": True}),
),
HttpCompressionMiddleware,
)
@coroutine_test
async def test_no_crawler_constructor(self):
with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"):
mw = HttpCompressionMiddleware()
buf = BytesIO()
with GzipFile(fileobj=buf, mode="wb") as f:
f.write(b"hello")
body = buf.getvalue()
request = Request("http://scrapytest.org")
response = Response(
"http://scrapytest.org",
body=body,
headers={"Content-Encoding": "gzip"},
)
newresponse = await mw.process_response(request, response)
assert newresponse.body == b"hello"
def test_process_request(self):
request = Request("http://scrapytest.org")
assert "Accept-Encoding" not in request.headers
self.mw.process_request(request)
assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)
def test_process_request_body_file(self):
request = Request("https://example.com", meta={"body_file": "body"})
self.mw.process_request(request)
assert "Accept-Encoding" not in request.headers
@coroutine_test
async def test_process_response_body_file(self):
request = Request("https://example.com", meta={"body_file": "body"})
response = BodyFileResponse(
"https://example.com",
headers={"Content-Encoding": "gzip"},
body_file="body",
request=request,
)
assert await self.mw.process_response(request, response) is response
@coroutine_test
async def test_process_response_gzip(self):
response = self._getresponse("gzip")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@coroutine_test
async def test_process_response_br(self):
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@coroutine_test
async def test_process_response_zstd(self):
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
continue
response = self._getresponse(check_key)
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
newresponse = await self.mw.process_response(request, response)
if raw_content is None:
raw_content = newresponse.body
else:
assert raw_content == newresponse.body
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
@coroutine_test
async def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
@coroutine_test
async def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
@coroutine_test
async def test_process_response_plain(self):
response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
request = Request("http://scrapytest.org")
assert not response.headers.get("Content-Encoding")
newresponse = await self.mw.process_response(request, response)
assert newresponse is response
assert newresponse.body.startswith(b"<!DOCTYPE")
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
@coroutine_test
async def test_multipleencodings(self):
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]
@coroutine_test
async def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
newresponse = await self._mw(2).process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@coroutine_test
async def test_multi_compression_single_header_invalid_compression(
self, caplog: pytest.LogCaptureFixture
) -> None:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
assert response.request
request = response.request
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
newresponse = await self.mw.process_response(request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
(
"HttpCompressionMiddleware cannot decode the response for"
" http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."
),
),
]
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@coroutine_test
async def test_multi_compression_multiple_header(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
assert response.request
request = response.request
newresponse = await self._mw(2).process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@coroutine_test
async def test_multi_compression_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@coroutine_test
async def test_multi_compression_single_and_multiple_header(self):
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
assert response.request
request = response.request
newresponse = await self._mw(3).process_response(request, response)
assert newresponse is not response
assert "Content-Encoding" not in newresponse.headers
assert newresponse.body.startswith(b"<!DOCTYPE")
@coroutine_test
async def test_multi_compression_single_and_multiple_header_invalid_compression(
self,
):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
@coroutine_test
async def test_max_encodings_default(
self, caplog: pytest.LogCaptureFixture
) -> None:
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
with (
caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
),
pytest.raises(IgnoreRequest),
):
await self.mw.process_response(request, response)
assert "COMPRESSION_MAX_ENCODINGS (1)" in caplog.text
@coroutine_test
async def test_max_encodings_rejects_before_decompressing(self) -> None:
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = [b", ".join([b"gzip"] * 5000)]
assert response.request
request = response.request
with pytest.raises(IgnoreRequest, match="lists 5000 supported encodings"):
await self.mw.process_response(request, response)
@coroutine_test
async def test_process_response_encoding_inside_body(self):
headers = {
"Content-Type": "text/html",
"Content-Encoding": "gzip",
}
f = BytesIO()
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
)
zf = GzipFile(fileobj=f, mode="wb")
zf.write(plainbody)
zf.close()
response = Response(
"http://www.example.com/", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
@coroutine_test
async def test_process_response_force_recalculate_encoding(self):
headers = {
"Content-Type": "text/html",
"Content-Encoding": "gzip",
}
f = BytesIO()
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
)
zf = GzipFile(fileobj=f, mode="wb")
zf.write(plainbody)
zf.close()
response = HtmlResponse(
"http://www.example.com/page.html", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
@coroutine_test
async def test_process_response_no_content_type_header(self):
headers = {b"Content-Encoding": b"identity"}
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
)
respcls = responsetypes.from_args(
url="http://www.example.com/index", headers=headers, body=plainbody
)
response = respcls(
"http://www.example.com/index", headers=headers, body=plainbody
)
request = Request("http://www.example.com/index")
newresponse = await self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
@coroutine_test
async def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@coroutine_test
async def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/octet-stream"
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@coroutine_test
async def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
response.headers["Content-Type"] = "binary/octet-stream"
assert response.request
request = response.request
newresponse = await self.mw.process_response(request, response)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@coroutine_test
async def test_process_response_gzipped_gzip_file(self):
"""Test that a gzip Content-Encoded .gz file is gunzipped
only once by the middleware, leaving gunzipping of the file
to upper layers.
"""
headers = {
"Content-Type": "application/gzip",
"Content-Encoding": "gzip",
}
# build a gzipped file (here, a sitemap)
f = BytesIO()
plainbody = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url>
<loc>http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc>http://www.example.com/Special-Offers.html</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
</urlset>"""
gz_file = GzipFile(fileobj=f, mode="wb")
gz_file.write(plainbody)
gz_file.close()
# build a gzipped response body containing this gzipped file
r = BytesIO()
gz_resp = GzipFile(fileobj=r, mode="wb")
gz_resp.write(f.getvalue())
gz_resp.close()
response = Response(
"http://www.example.com/", headers=headers, body=r.getvalue()
)
request = Request("http://www.example.com/")
newresponse = await self.mw.process_response(request, response)
assert gunzip(newresponse.body) == plainbody
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", len(f.getvalue()))
@coroutine_test
async def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
newresponse = await self.mw.process_response(request, response)
assert newresponse is response
assert response.body == b""
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
@pytest.mark.parametrize(
"extra",
[
pytest.param(b"garbage", id="trailing-garbage"),
pytest.param(zlib.compress(b"b" * 100_000), id="second-zlib-stream"),
],
)
def test_deflate_extra_data_after_stream(self, extra: bytes) -> None:
"""Bytes after the end of a deflate stream must not stall decompression.
Such bytes stay in ``unconsumed_tail`` and produce no output, so feeding
them back to the decompressor never makes progress. Decompressing in a
thread lets the test fail instead of hanging if that ever regresses.
"""
# The body must decompress to more than _CHUNK_SIZE, or the whole stream
# is consumed by the first decompress() call and no loop is entered.
plain = b"a" * 100_000
response = Response(
"http://example.com",
body=zlib.compress(plain) + extra,
headers={"Content-Encoding": "deflate"},
)
result: list[Request | Response] = []
thread = Thread(
target=lambda: result.append(
asyncio.run(
self.mw.process_response(Request("http://example.com"), response)
)
),
daemon=True,
)
thread.start()
thread.join(60)
assert not thread.is_alive(), "decompression did not terminate"
assert result[0].body == plain
@pytest.mark.parametrize(
"size", [_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE]
)
def test_deflate_chunked_output_leaves_nothing_to_flush(self, size: int) -> None:
"""Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``.
``decompress(max_length=...)`` leaves the input it could not turn into
output in ``unconsumed_tail``, so once that is empty every complete
stream has been fully emitted. If a future zlib buffers output instead,
this fails and ``_inflate()`` needs to flush the remainder again.
"""
decompressor = zlib.decompressobj()
decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)
while decompressor.unconsumed_tail and not decompressor.eof:
decompressor.decompress(
decompressor.unconsumed_tail, max_length=_CHUNK_SIZE
)
assert decompressor.flush() == b""
async def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
assert str(cause) == (
f"The number of bytes decompressed so far ({cause.decompressed_size} B) "
"exceeded the specified maximum (1000000 B)."
)
@coroutine_test
async def test_compression_bomb_setting_br(self):
await self._test_compression_bomb_setting("br")
@coroutine_test
async def test_compression_bomb_setting_deflate(self):
await self._test_compression_bomb_setting("deflate")
@coroutine_test
async def test_compression_bomb_setting_gzip(self):
await self._test_compression_bomb_setting("gzip")
@coroutine_test
async def test_compression_bomb_setting_zstd(self):
await self._test_compression_bomb_setting("zstd")
@coroutine_test
async def test_compression_bomb_setting_logs_warning(self, caplog):
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
assert response.request
caplog.clear()
with (
caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
),
pytest.raises(IgnoreRequest) as exc_info,
):
await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
str(exc_info.value),
)
]
async def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000
crawler = get_crawler(DownloadMaxSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_compression_bomb_spider_attr_br(self):
await self._test_compression_bomb_spider_attr("br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_compression_bomb_spider_attr_deflate(self):
await self._test_compression_bomb_spider_attr("deflate")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_compression_bomb_spider_attr_gzip(self):
await self._test_compression_bomb_spider_attr("gzip")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_compression_bomb_spider_attr_zstd(self):
await self._test_compression_bomb_spider_attr("zstd")
async def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 1_000_000
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
await mw.process_response(response.request, response)
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@coroutine_test
async def test_compression_bomb_request_meta_br(self):
await self._test_compression_bomb_request_meta("br")
@coroutine_test
async def test_compression_bomb_request_meta_deflate(self):
await self._test_compression_bomb_request_meta("deflate")
@coroutine_test
async def test_compression_bomb_request_meta_gzip(self):
await self._test_compression_bomb_request_meta("gzip")
@coroutine_test
async def test_compression_bomb_request_meta_zstd(self):
await self._test_compression_bomb_request_meta("zstd")
async def _test_download_warnsize_setting(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
]
@coroutine_test
async def test_download_warnsize_setting_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_setting(caplog, "br")
@coroutine_test
async def test_download_warnsize_setting_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_setting(caplog, "deflate")
@coroutine_test
async def test_download_warnsize_setting_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_setting(caplog, "gzip")
@coroutine_test
async def test_download_warnsize_setting_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_setting(caplog, "zstd")
async def _test_download_warnsize_spider_attr(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
class DownloadWarnSizeSpider(Spider):
download_warnsize = 10_000_000
crawler = get_crawler(DownloadWarnSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_download_warnsize_spider_attr_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_spider_attr(caplog, "br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_download_warnsize_spider_attr_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_spider_attr(caplog, "deflate")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_download_warnsize_spider_attr_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_spider_attr(caplog, "gzip")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@coroutine_test
async def test_download_warnsize_spider_attr_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_spider_attr(caplog, "zstd")
async def _test_download_warnsize_request_meta(
self, caplog: pytest.LogCaptureFixture, compression_id: str
) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_warnsize"] = 10_000_000
assert response.request
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
await mw.process_response(response.request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
]
@coroutine_test
async def test_download_warnsize_setting_ignored_with_request_maxsize(
self, caplog: pytest.LogCaptureFixture
) -> None:
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse("bomb-gzip")
response.meta["download_maxsize"] = 20_000_000
assert response.request
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
await mw.process_response(response.request, response)
assert not caplog.records
@coroutine_test
async def test_download_warnsize_request_meta_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_request_meta(caplog, "br")
@coroutine_test
async def test_download_warnsize_request_meta_deflate(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_request_meta(caplog, "deflate")
@coroutine_test
async def test_download_warnsize_request_meta_gzip(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_request_meta(caplog, "gzip")
@coroutine_test
async def test_download_warnsize_request_meta_zstd(
self, caplog: pytest.LogCaptureFixture
) -> None:
await self._test_download_warnsize_request_meta(caplog, "zstd")
async def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]
response = response.replace(body=truncated_body)
assert response.request
new_response = await mw.process_response(response.request, response)
assert isinstance(new_response, Response)
return new_response
@coroutine_test
async def test_process_truncated_response_br(self):
resp = await self._get_truncated_response("br")
assert resp.body.startswith(b"<!DOCTYPE")
@coroutine_test
async def test_process_truncated_response_zlibdeflate(self):
resp = await self._get_truncated_response("zlibdeflate")
assert resp.body.startswith(b"<!DOCTYPE")
@coroutine_test
async def test_process_truncated_response_gzip(self):
resp = await self._get_truncated_response("gzip")
assert resp.body.startswith(b"<!DOCTYPE")
@pytest.mark.parametrize(
"check_key", [key for key in FORMAT if key.startswith("zstd-")]
)
@coroutine_test
async def test_process_truncated_response_zstd(self, check_key: str) -> None:
resp = await self._get_truncated_response(check_key)
assert len(resp.body) == 0
@pytest.mark.parametrize(
("body", "encoding"),
[
(b"not compressed", "gzip"),
(b"\x1f\x8b\x08\x00", "gzip"),
(b"\x1f\x8b\x08" + b"\x00" * 7 + b"\xff" * 20, "gzip"),
(b"not compressed", "deflate"),
(b"not compressed", "br"),
(b"not compressed", "zstd"),
],
)
@coroutine_test
async def test_process_response_invalid(self, body: bytes, encoding: str) -> None:
request = Request("http://example.com")
response = Response(
"http://example.com", body=body, headers={"Content-Encoding": encoding}
)
with pytest.raises(DecompressionError):
await self.mw.process_response(request, response)