Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not decode of head response (2)
omerffa77e1ac60f
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 7ab304c17..bcf20f10c 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -20,6 +20,9 @@ class HttpCompressionMiddleware(object):request.headers.setdefault('Accept-Encoding', 'gzip,deflate')def process_response(self, request, response, spider):++ if request.method == 'HEAD':+ return responseif isinstance(response, Response):content_encoding = response.headers.getlist('Content-Encoding')if content_encoding and not is_gzipped(response):