Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[HttpCompressionMiddleware] Do not decompress binary/octet-stream responses

Paul Tremberth10a2c46e12e5
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index d035f9fdf..b2f737193 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -53,5 +53,10 @@ def gunzip(data):
def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
- ctype = response.headers.get('Content-Type', b'')
- return ctype in (b'application/x-gzip', b'application/gzip')
+ ctype = response.headers.get('Content-Type', b'').lower()
+ cenc = response.headers.get('Content-Encoding', b'').lower()
+ return (
+ ctype in (b'application/x-gzip', b'application/gzip') or
+ (ctype in (b'application/octet-stream', b'binary/octet-stream') and
+ cenc in (b'gzip', b'x-gzip'))
+ )
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 24955a515..b2426946d 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase):
self.assertEqual(response.headers['Content-Encoding'], b'gzip')
self.assertEqual(response.headers['Content-Type'], b'application/gzip')
+ def test_process_response_gzip_app_octetstream_contenttype(self):
+ response = self._getresponse('gzip')
+ response.headers['Content-Type'] = 'application/octet-stream'
+ request = response.request
+
+ newresponse = self.mw.process_response(request, response, self.spider)
+ self.assertIs(newresponse, response)
+ self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+ self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')
+
+ def test_process_response_gzip_binary_octetstream_contenttype(self):
+ response = self._getresponse('x-gzip')
+ response.headers['Content-Type'] = 'binary/octet-stream'
+ request = response.request
+
+ newresponse = self.mw.process_response(request, response, self.spider)
+ self.assertIs(newresponse, response)
+ self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+ self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')
+
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse('gzip')
response.headers['Content-Type'] = 'application/gzip'