Scrapy, a fast high-level web crawling & scraping framework for Python.
Mind Spider.download_maxsize and Request.meta['download_maxsize']
Adrián Chavese0b66c021ae2
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 8ee1d95a6..e6463307e 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,5 +1,6 @@import warnings+from scrapy import signalsfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes@@ -38,6 +39,7 @@ class HttpCompressionMiddleware:returnself.stats = crawler.statsself._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")+ crawler.signals.connect(self.open_spider, signals.spider_opened)@classmethoddef from_crawler(cls, crawler):@@ -52,10 +54,15 @@ class HttpCompressionMiddleware:"reimplement their 'from_crawler' method.",ScrapyDeprecationWarning,)- result = cls()- result.stats = crawler.stats- result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")- return result+ spider = cls()+ spider.stats = crawler.stats+ spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")+ crawler.signals.connect(spider.open_spider, signals.spider_opened)+ return spider++ def open_spider(self, spider):+ if hasattr(spider, "download_maxsize"):+ self._max_size = spider.download_maxsizedef process_request(self, request, spider):request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))@@ -67,8 +74,11 @@ class HttpCompressionMiddleware:content_encoding = response.headers.getlist("Content-Encoding")if content_encoding:encoding = content_encoding.pop()+ max_size = request.meta.get("download_maxsize", self._max_size)try:- decoded_body = self._decode(response.body, encoding.lower())+ decoded_body = self._decode(+ response.body, encoding.lower(), max_size+ )except _DecompressionMaxSizeExceeded:raise IgnoreRequest(f"Ignored response {response} because its body "@@ -98,13 +108,13 @@ class HttpCompressionMiddleware:return response- def _decode(self, body, encoding):+ def _decode(self, body, encoding, max_size):if encoding == b"gzip" or encoding == b"x-gzip":- return gunzip(body, max_size=self._max_size)+ return gunzip(body, max_size=max_size)if encoding == b"deflate":- return _inflate(body, max_size=self._max_size)+ return _inflate(body, max_size=max_size)if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:- return _unbrotli(body, max_size=self._max_size)+ return _unbrotli(body, max_size=max_size)if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:- return _unzstd(body, max_size=self._max_size)+ return _unzstd(body, max_size=max_size)return bodydiff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex cc8b13cc3..3bca3f5c2 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -72,10 +72,14 @@ class SitemapSpider(Spider):if isinstance(response, XmlResponse):return response.bodyif gzip_magic_number(response):+ max_size = response.meta.get(+ "download_maxsize",+ getattr(+ self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE")+ ),+ )try:- return gunzip(- response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE")- )+ return gunzip(response.body, max_size=max_size)except _DecompressionMaxSizeExceeded:return None# actual gzipped sitemap files are decompressed above ;diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 7babd1318..6d71ba71e 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -49,10 +49,7 @@ FORMAT = {class HttpCompressionTest(TestCase):def setUp(self):- settings = {- "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests.- }- self.crawler = get_crawler(Spider, settings_dict=settings)+ self.crawler = get_crawler(Spider)self.spider = self.crawler._create_spider("scrapytest.org")self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)self.crawler.stats.open_spider(self.spider)@@ -373,31 +370,103 @@ class HttpCompressionTest(TestCase):self.assertStatsEqual("httpcompression/response_count", None)self.assertStatsEqual("httpcompression/response_bytes", None)- def _test_compression_bomb(self, compression_id):+ def _test_compression_bomb_setting(self, compression_id):+ settings = {"DOWNLOAD_MAXSIZE": 10_000_000}+ crawler = get_crawler(Spider, settings_dict=settings)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)++ response = self._getresponse(f"bomb-{compression_id}")+ self.assertRaises(+ IgnoreRequest,+ mw.process_response,+ response.request,+ response,+ spider,+ )++ def test_compression_bomb_setting_br(self):+ try:+ import brotli # noqa: F401+ except ImportError:+ raise SkipTest("no brotli")+ self._test_compression_bomb_setting("br")++ def test_compression_bomb_setting_deflate(self):+ self._test_compression_bomb_setting("deflate")++ def test_compression_bomb_setting_gzip(self):+ self._test_compression_bomb_setting("gzip")++ def test_compression_bomb_setting_zstd(self):+ self._test_compression_bomb_setting("zstd")++ def _test_compression_bomb_spider_attr(self, compression_id):+ class DownloadMaxSizeSpider(Spider):+ download_maxsize = 10_000_000++ crawler = get_crawler(DownloadMaxSizeSpider)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)++ response = self._getresponse(f"bomb-{compression_id}")+ self.assertRaises(+ IgnoreRequest,+ mw.process_response,+ response.request,+ response,+ spider,+ )++ def test_compression_bomb_spider_attr_br(self):+ try:+ import brotli # noqa: F401+ except ImportError:+ raise SkipTest("no brotli")+ self._test_compression_bomb_spider_attr("br")++ def test_compression_bomb_spider_attr_deflate(self):+ self._test_compression_bomb_spider_attr("deflate")++ def test_compression_bomb_spider_attr_gzip(self):+ self._test_compression_bomb_spider_attr("gzip")++ def test_compression_bomb_spider_attr_zstd(self):+ self._test_compression_bomb_spider_attr("zstd")++ def _test_compression_bomb_request_meta(self, compression_id):+ crawler = get_crawler(Spider)+ spider = crawler._create_spider("scrapytest.org")+ mw = HttpCompressionMiddleware.from_crawler(crawler)+ mw.open_spider(spider)+response = self._getresponse(f"bomb-{compression_id}")+ response.meta["download_maxsize"] = 10_000_000self.assertRaises(IgnoreRequest,- self.mw.process_response,+ mw.process_response,response.request,response,- self.spider,+ spider,)- def test_compression_bomb_br(self):+ def test_compression_bomb_request_meta_br(self):try:import brotli # noqa: F401except ImportError:raise SkipTest("no brotli")- self._test_compression_bomb("br")+ self._test_compression_bomb_request_meta("br")- def test_compression_bomb_deflate(self):- self._test_compression_bomb("deflate")+ def test_compression_bomb_request_meta_deflate(self):+ self._test_compression_bomb_request_meta("deflate")- def test_compression_bomb_gzip(self):- self._test_compression_bomb("gzip")+ def test_compression_bomb_request_meta_gzip(self):+ self._test_compression_bomb_request_meta("gzip")- def test_compression_bomb_zstd(self):- self._test_compression_bomb("zstd")+ def test_compression_bomb_request_meta_zstd(self):+ self._test_compression_bomb_request_meta("zstd")class HttpCompressionSubclassTest(TestCase):diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 875ff5454..e8480ceb4 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -509,6 +509,7 @@ class SitemapSpiderTest(SpiderTest):url="http://www.example.com/sitemap",body=self.GZBODY,headers={"content-type": "application/gzip"},+ request=Request("http://www.example.com/sitemap"),)self.assertSitemapBody(r, self.BODY)@@ -517,7 +518,11 @@ class SitemapSpiderTest(SpiderTest):self.assertSitemapBody(r, self.BODY)def test_get_sitemap_body_xml_url_compressed(self):- r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)+ r = Response(+ url="http://www.example.com/sitemap.xml.gz",+ body=self.GZBODY,+ request=Request("http://www.example.com/sitemap"),+ )self.assertSitemapBody(r, self.BODY)# .xml.gz but body decoded by HttpCompression middleware already@@ -694,13 +699,37 @@ Sitemap: /sitemap-relative-url.xml["http://www.example.com/sitemap2.xml"],)- def test_compression_bomb(self):+ def test_compression_bomb_setting(self):settings = {"DOWNLOAD_MAXSIZE": 10_000_000}crawler = get_crawler(settings_dict=settings)spider = self.spider_class.from_crawler(crawler, "example.com")body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")body = body_path.read_bytes()- response = Response(url="https://example.com", body=body)+ request = Request(url="https://example.com")+ response = Response(url="https://example.com", body=body, request=request)+ self.assertIsNone(spider._get_sitemap_body(response))++ def test_compression_bomb_spider_attr(self):+ class DownloadMaxSizeSpider(self.spider_class):+ download_maxsize = 10_000_000++ crawler = get_crawler()+ spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")+ body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")+ body = body_path.read_bytes()+ request = Request(url="https://example.com")+ response = Response(url="https://example.com", body=body, request=request)+ self.assertIsNone(spider._get_sitemap_body(response))++ def test_compression_bomb_request_meta(self):+ crawler = get_crawler()+ spider = self.spider_class.from_crawler(crawler, "example.com")+ body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")+ body = body_path.read_bytes()+ request = Request(+ url="https://example.com", meta={"download_maxsize": 10_000_000}+ )+ response = Response(url="https://example.com", body=body, request=request)self.assertIsNone(spider._get_sitemap_body(response))