Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618)

Andrey Rakhmatullin391af6afcca2
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index a65757972..58891b952 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -111,6 +111,8 @@ class HttpCompressionMiddleware:
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
+ if content_encoding:
+ self._warn_unknown_encoding(response, content_encoding)
response.headers["Content-Encoding"] = content_encoding
if self.stats:
self.stats.inc_value(
@@ -143,9 +145,11 @@ class HttpCompressionMiddleware:
body = self._decode(body, encoding, max_size)
return body, to_keep
+ @staticmethod
def _split_encodings(
- self, content_encoding: list[bytes]
+ content_encoding: list[bytes],
) -> tuple[list[bytes], list[bytes]]:
+ supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"}
to_keep: list[bytes] = [
encoding.strip().lower()
for encoding in chain.from_iterable(
@@ -155,19 +159,35 @@ class HttpCompressionMiddleware:
to_decode: list[bytes] = []
while to_keep:
encoding = to_keep.pop()
- if encoding not in ACCEPTED_ENCODINGS:
+ if encoding not in supported_encodings:
to_keep.append(encoding)
return to_decode, to_keep
to_decode.append(encoding)
return to_decode, to_keep
- def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
+ @staticmethod
+ def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes:
if encoding in {b"gzip", b"x-gzip"}:
return gunzip(body, max_size=max_size)
if encoding == b"deflate":
return _inflate(body, max_size=max_size)
- if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
+ if encoding == b"br":
return _unbrotli(body, max_size=max_size)
- if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
+ if encoding == b"zstd":
return _unzstd(body, max_size=max_size)
- return body
+ # shouldn't be reached
+ return body # pragma: no cover
+
+ def _warn_unknown_encoding(
+ self, response: Response, encodings: list[bytes]
+ ) -> None:
+ encodings_str = b",".join(encodings).decode()
+ msg = (
+ f"{self.__class__.__name__} cannot decode the response for {response.url} "
+ f"from unsupported encoding(s) '{encodings_str}'."
+ )
+ if b"br" in encodings:
+ msg += " You need to install brotli or brotlicffi to decode 'br'."
+ if b"zstd" in encodings:
+ msg += " You need to install zstandard to decode 'zstd'."
+ logger.warning(msg)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 78d0dd99d..a1c5883ec 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed")
FORMAT = {
"gzip": ("html-gzip.bin", "gzip"),
- "x-gzip": ("html-gzip.bin", "gzip"),
+ "x-gzip": ("html-gzip.bin", "x-gzip"),
"rawdeflate": ("html-rawdeflate.bin", "deflate"),
"zlibdeflate": ("html-zlibdeflate.bin", "deflate"),
"gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"),
@@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase):
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
+ def test_process_response_br_unsupported(self):
+ try:
+ try:
+ import brotli # noqa: F401
+
+ raise SkipTest("Requires not having brotli support")
+ except ImportError:
+ import brotlicffi # noqa: F401
+
+ raise SkipTest("Requires not having brotli support")
+ except ImportError:
+ pass
+ response = self._getresponse("br")
+ request = response.request
+ self.assertEqual(response.headers["Content-Encoding"], b"br")
+ with LogCapture(
+ "scrapy.downloadermiddlewares.httpcompression",
+ propagate=False,
+ level=WARNING,
+ ) as log:
+ newresponse = self.mw.process_response(request, response, self.spider)
+ log.check(
+ (
+ "scrapy.downloadermiddlewares.httpcompression",
+ "WARNING",
+ (
+ "HttpCompressionMiddleware cannot decode the response for"
+ " http://scrapytest.org/ from unsupported encoding(s) 'br'."
+ " You need to install brotli or brotlicffi to decode 'br'."
+ ),
+ ),
+ )
+ assert newresponse is not response
+ self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"])
+
def test_process_response_zstd(self):
try:
import zstandard # noqa: F401
@@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase):
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
+ def test_process_response_zstd_unsupported(self):
+ try:
+ import zstandard # noqa: F401
+
+ raise SkipTest("Requires not having zstandard support")
+ except ImportError:
+ pass
+ response = self._getresponse("zstd-static-content-size")
+ request = response.request
+ self.assertEqual(response.headers["Content-Encoding"], b"zstd")
+ with LogCapture(
+ "scrapy.downloadermiddlewares.httpcompression",
+ propagate=False,
+ level=WARNING,
+ ) as log:
+ newresponse = self.mw.process_response(request, response, self.spider)
+ log.check(
+ (
+ "scrapy.downloadermiddlewares.httpcompression",
+ "WARNING",
+ (
+ "HttpCompressionMiddleware cannot decode the response for"
+ " http://scrapytest.org/ from unsupported encoding(s) 'zstd'."
+ " You need to install zstandard to decode 'zstd'."
+ ),
+ ),
+ )
+ assert newresponse is not response
+ self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"zstd"])
+
def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
request = response.request
@@ -221,7 +286,22 @@ class HttpCompressionTest(TestCase):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
request = response.request
- newresponse = self.mw.process_response(request, response, self.spider)
+ with LogCapture(
+ "scrapy.downloadermiddlewares.httpcompression",
+ propagate=False,
+ level=WARNING,
+ ) as log:
+ newresponse = self.mw.process_response(request, response, self.spider)
+ log.check(
+ (
+ "scrapy.downloadermiddlewares.httpcompression",
+ "WARNING",
+ (
+ "HttpCompressionMiddleware cannot decode the response for"
+ " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."
+ ),
+ ),
+ )
assert newresponse is not response
self.assertEqual(
newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"]