Scrapy, a fast high-level web crawling & scraping framework for Python.
[HttpCompressionMiddleware] Do not decompress binary/octet-stream responses
Paul Tremberth10a2c46e12e5
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex d035f9fdf..b2f737193 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -53,5 +53,10 @@ def gunzip(data):def is_gzipped(response):"""Return True if the response is gzipped, or False otherwise"""- ctype = response.headers.get('Content-Type', b'')- return ctype in (b'application/x-gzip', b'application/gzip')+ ctype = response.headers.get('Content-Type', b'').lower()+ cenc = response.headers.get('Content-Encoding', b'').lower()+ return (+ ctype in (b'application/x-gzip', b'application/gzip') or+ (ctype in (b'application/octet-stream', b'binary/octet-stream') and+ cenc in (b'gzip', b'x-gzip'))+ )diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 24955a515..b2426946d 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase):self.assertEqual(response.headers['Content-Encoding'], b'gzip')self.assertEqual(response.headers['Content-Type'], b'application/gzip')+ def test_process_response_gzip_app_octetstream_contenttype(self):+ response = self._getresponse('gzip')+ response.headers['Content-Type'] = 'application/octet-stream'+ request = response.request++ newresponse = self.mw.process_response(request, response, self.spider)+ self.assertIs(newresponse, response)+ self.assertEqual(response.headers['Content-Encoding'], b'gzip')+ self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')++ def test_process_response_gzip_binary_octetstream_contenttype(self):+ response = self._getresponse('x-gzip')+ response.headers['Content-Type'] = 'binary/octet-stream'+ request = response.request++ newresponse = self.mw.process_response(request, response, self.spider)+ self.assertIs(newresponse, response)+ self.assertEqual(response.headers['Content-Encoding'], b'gzip')+ self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')+def test_process_response_head_request_no_decode_required(self):response = self._getresponse('gzip')response.headers['Content-Type'] = 'application/gzip'