Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'origin/master' into octet-stream-no-decompress
Paul Tremberth778f1cf84cfc
diff --cc scrapy/utils/gz.pyindex b2f737193,cfb652143..9fe88d108--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@@ -50,13 -50,9 +50,13 @@@ def gunzip(data)raisereturn output+ _is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)++_is_octetstream_re = re.compile(br'^(application|binary)/octet-stream\b', re.I)def is_gzipped(response):"""Return True if the response is gzipped, or False otherwise"""- ctype = response.headers.get('Content-Type', b'').lower()+ ctype = response.headers.get('Content-Type', b'')- return _is_gzipped_re.search(ctype) is not None+ cenc = response.headers.get('Content-Encoding', b'').lower()- return (- ctype in (b'application/x-gzip', b'application/gzip') or- (ctype in (b'application/octet-stream', b'binary/octet-stream') and- cenc in (b'gzip', b'x-gzip'))- )++ return (_is_gzipped_re.search(ctype) is not None or++ (_is_octetstream_re.search(ctype) is not None and++ cenc in (b'gzip', b'x-gzip')))