Scrapy, a fast high-level web crawling & scraping framework for Python.
Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618)
Andrey Rakhmatullin391af6afcca2
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex a65757972..58891b952 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -111,6 +111,8 @@ class HttpCompressionMiddleware:f"({len(decoded_body)} B) is larger than the "f"download warning size ({warn_size} B).")+ if content_encoding:+ self._warn_unknown_encoding(response, content_encoding)response.headers["Content-Encoding"] = content_encodingif self.stats:self.stats.inc_value(@@ -143,9 +145,11 @@ class HttpCompressionMiddleware:body = self._decode(body, encoding, max_size)return body, to_keep+ @staticmethoddef _split_encodings(- self, content_encoding: list[bytes]+ content_encoding: list[bytes],) -> tuple[list[bytes], list[bytes]]:+ supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"}to_keep: list[bytes] = [encoding.strip().lower()for encoding in chain.from_iterable(@@ -155,19 +159,35 @@ class HttpCompressionMiddleware:to_decode: list[bytes] = []while to_keep:encoding = to_keep.pop()- if encoding not in ACCEPTED_ENCODINGS:+ if encoding not in supported_encodings:to_keep.append(encoding)return to_decode, to_keepto_decode.append(encoding)return to_decode, to_keep- def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:+ @staticmethod+ def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes:if encoding in {b"gzip", b"x-gzip"}:return gunzip(body, max_size=max_size)if encoding == b"deflate":return _inflate(body, max_size=max_size)- if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:+ if encoding == b"br":return _unbrotli(body, max_size=max_size)- if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:+ if encoding == b"zstd":return _unzstd(body, max_size=max_size)- return body+ # shouldn't be reached+ return body # pragma: no cover++ def _warn_unknown_encoding(+ self, response: Response, encodings: list[bytes]+ ) -> None:+ encodings_str = b",".join(encodings).decode()+ msg = (+ f"{self.__class__.__name__} cannot decode the response for {response.url} "+ f"from unsupported encoding(s) '{encodings_str}'."+ )+ if b"br" in encodings:+ msg += " You need to install brotli or brotlicffi to decode 'br'."+ if b"zstd" in encodings:+ msg += " You need to install zstandard to decode 'zstd'."+ logger.warning(msg)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 78d0dd99d..a1c5883ec 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed")FORMAT = {"gzip": ("html-gzip.bin", "gzip"),- "x-gzip": ("html-gzip.bin", "gzip"),+ "x-gzip": ("html-gzip.bin", "x-gzip"),"rawdeflate": ("html-rawdeflate.bin", "deflate"),"zlibdeflate": ("html-zlibdeflate.bin", "deflate"),"gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"),@@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase):self.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 74837)+ def test_process_response_br_unsupported(self):+ try:+ try:+ import brotli # noqa: F401++ raise SkipTest("Requires not having brotli support")+ except ImportError:+ import brotlicffi # noqa: F401++ raise SkipTest("Requires not having brotli support")+ except ImportError:+ pass+ response = self._getresponse("br")+ request = response.request+ self.assertEqual(response.headers["Content-Encoding"], b"br")+ with LogCapture(+ "scrapy.downloadermiddlewares.httpcompression",+ propagate=False,+ level=WARNING,+ ) as log:+ newresponse = self.mw.process_response(request, response, self.spider)+ log.check(+ (+ "scrapy.downloadermiddlewares.httpcompression",+ "WARNING",+ (+ "HttpCompressionMiddleware cannot decode the response for"+ " http://scrapytest.org/ from unsupported encoding(s) 'br'."+ " You need to install brotli or brotlicffi to decode 'br'."+ ),+ ),+ )+ assert newresponse is not response+ self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"])+def test_process_response_zstd(self):try:import zstandard # noqa: F401@@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase):assert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers+ def test_process_response_zstd_unsupported(self):+ try:+ import zstandard # noqa: F401++ raise SkipTest("Requires not having zstandard support")+ except ImportError:+ pass+ response = self._getresponse("zstd-static-content-size")+ request = response.request+ self.assertEqual(response.headers["Content-Encoding"], b"zstd")+ with LogCapture(+ "scrapy.downloadermiddlewares.httpcompression",+ propagate=False,+ level=WARNING,+ ) as log:+ newresponse = self.mw.process_response(request, response, self.spider)+ log.check(+ (+ "scrapy.downloadermiddlewares.httpcompression",+ "WARNING",+ (+ "HttpCompressionMiddleware cannot decode the response for"+ " http://scrapytest.org/ from unsupported encoding(s) 'zstd'."+ " You need to install zstandard to decode 'zstd'."+ ),+ ),+ )+ assert newresponse is not response+ self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"zstd"])+def test_process_response_rawdeflate(self):response = self._getresponse("rawdeflate")request = response.request@@ -221,7 +286,22 @@ class HttpCompressionTest(TestCase):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ with LogCapture(+ "scrapy.downloadermiddlewares.httpcompression",+ propagate=False,+ level=WARNING,+ ) as log:+ newresponse = self.mw.process_response(request, response, self.spider)+ log.check(+ (+ "scrapy.downloadermiddlewares.httpcompression",+ "WARNING",+ (+ "HttpCompressionMiddleware cannot decode the response for"+ " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."+ ),+ ),+ )assert newresponse is not responseself.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"]