Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into octet-stream-no-decompress

Paul Tremberth778f1cf84cfc
diff --cc scrapy/utils/gz.py
index b2f737193,cfb652143..9fe88d108
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@@ -50,13 -50,9 +50,13 @@@ def gunzip(data)
raise
return output
+ _is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
++_is_octetstream_re = re.compile(br'^(application|binary)/octet-stream\b', re.I)
def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
- ctype = response.headers.get('Content-Type', b'').lower()
+ ctype = response.headers.get('Content-Type', b'')
- return _is_gzipped_re.search(ctype) is not None
+ cenc = response.headers.get('Content-Encoding', b'').lower()
- return (
- ctype in (b'application/x-gzip', b'application/gzip') or
- (ctype in (b'application/octet-stream', b'binary/octet-stream') and
- cenc in (b'gzip', b'x-gzip'))
- )
++ return (_is_gzipped_re.search(ctype) is not None or
++ (_is_octetstream_re.search(ctype) is not None and
++ cenc in (b'gzip', b'x-gzip')))