Scrapy, a fast high-level web crawling & scraping framework for Python.
Protect against gzip bomb sitemaps
Adrián Chaves3fda2fe103da
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex aaf75a519..cc8b13cc3 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -3,6 +3,7 @@ import refrom scrapy.http import Request, XmlResponsefrom scrapy.spiders import Spider+from scrapy.utils._compression import _DecompressionMaxSizeExceededfrom scrapy.utils.gz import gunzip, gzip_magic_numberfrom scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots@@ -71,7 +72,12 @@ class SitemapSpider(Spider):if isinstance(response, XmlResponse):return response.bodyif gzip_magic_number(response):- return gunzip(response.body)+ try:+ return gunzip(+ response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE")+ )+ except _DecompressionMaxSizeExceeded:+ return None# actual gzipped sitemap files are decompressed above ;# if we are here (response body is not gzipped)# and have a response for .xml.gz,diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 00da3d485..875ff5454 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -2,6 +2,7 @@ import gzipimport inspectimport warningsfrom io import BytesIO+from pathlib import Pathfrom typing import Anyfrom unittest import mock@@ -25,7 +26,7 @@ from scrapy.spiders import ()from scrapy.spiders.init import InitSpiderfrom scrapy.utils.test import get_crawler-from tests import get_testdata+from tests import get_testdata, tests_datadirclass SpiderTest(unittest.TestCase):@@ -489,7 +490,8 @@ class SitemapSpiderTest(SpiderTest):GZBODY = f.getvalue()def assertSitemapBody(self, response, body):- spider = self.spider_class("example.com")+ crawler = get_crawler()+ spider = self.spider_class.from_crawler(crawler, "example.com")self.assertEqual(spider._get_sitemap_body(response), body)def test_get_sitemap_body(self):@@ -692,6 +694,15 @@ Sitemap: /sitemap-relative-url.xml["http://www.example.com/sitemap2.xml"],)+ def test_compression_bomb(self):+ settings = {"DOWNLOAD_MAXSIZE": 10_000_000}+ crawler = get_crawler(settings_dict=settings)+ spider = self.spider_class.from_crawler(crawler, "example.com")+ body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")+ body = body_path.read_bytes()+ response = Response(url="https://example.com", body=body)+ self.assertIsNone(spider._get_sitemap_body(response))+class DeprecationTest(unittest.TestCase):def test_crawl_spider(self):