Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not silently fail on gzip unzipping
Paul Tremberthb174744b80fd
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex dd32c62de..203dee42d 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,6 +1,6 @@import zlib-from scrapy.utils.gz import gunzip, is_gzipped+from scrapy.utils.gz import gunzipfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.exceptions import NotConfigureddiff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 10af90259..e54001d88 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -61,10 +61,7 @@ class SitemapSpider(Spider):if isinstance(response, XmlResponse):return response.bodyelif gzip_magic_number(response):- try:- return gunzip(response.body)- except:- pass+ return gunzip(response.body)# actual gzipped sitemap files are decompressed above ;# if we are here (response body is not gzipped)# and have a response for .xml.gz,