Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Mind Spider.download_maxsize and Request.meta['download_maxsize']

Adrián Chavese0b66c021ae2
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 8ee1d95a6..e6463307e 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,5 +1,6 @@
import warnings
+from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
@@ -38,6 +39,7 @@ class HttpCompressionMiddleware:
return
self.stats = crawler.stats
self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+ crawler.signals.connect(self.open_spider, signals.spider_opened)
@classmethod
def from_crawler(cls, crawler):
@@ -52,10 +54,15 @@ class HttpCompressionMiddleware:
"reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
- result = cls()
- result.stats = crawler.stats
- result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
- return result
+ spider = cls()
+ spider.stats = crawler.stats
+ spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+ crawler.signals.connect(spider.open_spider, signals.spider_opened)
+ return spider
+
+ def open_spider(self, spider):
+ if hasattr(spider, "download_maxsize"):
+ self._max_size = spider.download_maxsize
def process_request(self, request, spider):
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
@@ -67,8 +74,11 @@ class HttpCompressionMiddleware:
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
encoding = content_encoding.pop()
+ max_size = request.meta.get("download_maxsize", self._max_size)
try:
- decoded_body = self._decode(response.body, encoding.lower())
+ decoded_body = self._decode(
+ response.body, encoding.lower(), max_size
+ )
except _DecompressionMaxSizeExceeded:
raise IgnoreRequest(
f"Ignored response {response} because its body "
@@ -98,13 +108,13 @@ class HttpCompressionMiddleware:
return response
- def _decode(self, body, encoding):
+ def _decode(self, body, encoding, max_size):
if encoding == b"gzip" or encoding == b"x-gzip":
- return gunzip(body, max_size=self._max_size)
+ return gunzip(body, max_size=max_size)
if encoding == b"deflate":
- return _inflate(body, max_size=self._max_size)
+ return _inflate(body, max_size=max_size)
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
- return _unbrotli(body, max_size=self._max_size)
+ return _unbrotli(body, max_size=max_size)
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
- return _unzstd(body, max_size=self._max_size)
+ return _unzstd(body, max_size=max_size)
return body
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index cc8b13cc3..3bca3f5c2 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -72,10 +72,14 @@ class SitemapSpider(Spider):
if isinstance(response, XmlResponse):
return response.body
if gzip_magic_number(response):
+ max_size = response.meta.get(
+ "download_maxsize",
+ getattr(
+ self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE")
+ ),
+ )
try:
- return gunzip(
- response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE")
- )
+ return gunzip(response.body, max_size=max_size)
except _DecompressionMaxSizeExceeded:
return None
# actual gzipped sitemap files are decompressed above ;
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 7babd1318..6d71ba71e 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -49,10 +49,7 @@ FORMAT = {
class HttpCompressionTest(TestCase):
def setUp(self):
- settings = {
- "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests.
- }
- self.crawler = get_crawler(Spider, settings_dict=settings)
+ self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider(self.spider)
@@ -373,31 +370,103 @@ class HttpCompressionTest(TestCase):
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
- def _test_compression_bomb(self, compression_id):
+ def _test_compression_bomb_setting(self, compression_id):
+ settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
+ crawler = get_crawler(Spider, settings_dict=settings)
+ spider = crawler._create_spider("scrapytest.org")
+ mw = HttpCompressionMiddleware.from_crawler(crawler)
+ mw.open_spider(spider)
+
+ response = self._getresponse(f"bomb-{compression_id}")
+ self.assertRaises(
+ IgnoreRequest,
+ mw.process_response,
+ response.request,
+ response,
+ spider,
+ )
+
+ def test_compression_bomb_setting_br(self):
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ raise SkipTest("no brotli")
+ self._test_compression_bomb_setting("br")
+
+ def test_compression_bomb_setting_deflate(self):
+ self._test_compression_bomb_setting("deflate")
+
+ def test_compression_bomb_setting_gzip(self):
+ self._test_compression_bomb_setting("gzip")
+
+ def test_compression_bomb_setting_zstd(self):
+ self._test_compression_bomb_setting("zstd")
+
+ def _test_compression_bomb_spider_attr(self, compression_id):
+ class DownloadMaxSizeSpider(Spider):
+ download_maxsize = 10_000_000
+
+ crawler = get_crawler(DownloadMaxSizeSpider)
+ spider = crawler._create_spider("scrapytest.org")
+ mw = HttpCompressionMiddleware.from_crawler(crawler)
+ mw.open_spider(spider)
+
+ response = self._getresponse(f"bomb-{compression_id}")
+ self.assertRaises(
+ IgnoreRequest,
+ mw.process_response,
+ response.request,
+ response,
+ spider,
+ )
+
+ def test_compression_bomb_spider_attr_br(self):
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ raise SkipTest("no brotli")
+ self._test_compression_bomb_spider_attr("br")
+
+ def test_compression_bomb_spider_attr_deflate(self):
+ self._test_compression_bomb_spider_attr("deflate")
+
+ def test_compression_bomb_spider_attr_gzip(self):
+ self._test_compression_bomb_spider_attr("gzip")
+
+ def test_compression_bomb_spider_attr_zstd(self):
+ self._test_compression_bomb_spider_attr("zstd")
+
+ def _test_compression_bomb_request_meta(self, compression_id):
+ crawler = get_crawler(Spider)
+ spider = crawler._create_spider("scrapytest.org")
+ mw = HttpCompressionMiddleware.from_crawler(crawler)
+ mw.open_spider(spider)
+
response = self._getresponse(f"bomb-{compression_id}")
+ response.meta["download_maxsize"] = 10_000_000
self.assertRaises(
IgnoreRequest,
- self.mw.process_response,
+ mw.process_response,
response.request,
response,
- self.spider,
+ spider,
)
- def test_compression_bomb_br(self):
+ def test_compression_bomb_request_meta_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
- self._test_compression_bomb("br")
+ self._test_compression_bomb_request_meta("br")
- def test_compression_bomb_deflate(self):
- self._test_compression_bomb("deflate")
+ def test_compression_bomb_request_meta_deflate(self):
+ self._test_compression_bomb_request_meta("deflate")
- def test_compression_bomb_gzip(self):
- self._test_compression_bomb("gzip")
+ def test_compression_bomb_request_meta_gzip(self):
+ self._test_compression_bomb_request_meta("gzip")
- def test_compression_bomb_zstd(self):
- self._test_compression_bomb("zstd")
+ def test_compression_bomb_request_meta_zstd(self):
+ self._test_compression_bomb_request_meta("zstd")
class HttpCompressionSubclassTest(TestCase):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 875ff5454..e8480ceb4 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -509,6 +509,7 @@ class SitemapSpiderTest(SpiderTest):
url="http://www.example.com/sitemap",
body=self.GZBODY,
headers={"content-type": "application/gzip"},
+ request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
@@ -517,7 +518,11 @@ class SitemapSpiderTest(SpiderTest):
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url_compressed(self):
- r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
+ r = Response(
+ url="http://www.example.com/sitemap.xml.gz",
+ body=self.GZBODY,
+ request=Request("http://www.example.com/sitemap"),
+ )
self.assertSitemapBody(r, self.BODY)
# .xml.gz but body decoded by HttpCompression middleware already
@@ -694,13 +699,37 @@ Sitemap: /sitemap-relative-url.xml
["http://www.example.com/sitemap2.xml"],
)
- def test_compression_bomb(self):
+ def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
- response = Response(url="https://example.com", body=body)
+ request = Request(url="https://example.com")
+ response = Response(url="https://example.com", body=body, request=request)
+ self.assertIsNone(spider._get_sitemap_body(response))
+
+ def test_compression_bomb_spider_attr(self):
+ class DownloadMaxSizeSpider(self.spider_class):
+ download_maxsize = 10_000_000
+
+ crawler = get_crawler()
+ spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")
+ body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
+ body = body_path.read_bytes()
+ request = Request(url="https://example.com")
+ response = Response(url="https://example.com", body=body, request=request)
+ self.assertIsNone(spider._get_sitemap_body(response))
+
+ def test_compression_bomb_request_meta(self):
+ crawler = get_crawler()
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+ body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
+ body = body_path.read_bytes()
+ request = Request(
+ url="https://example.com", meta={"download_maxsize": 10_000_000}
+ )
+ response = Response(url="https://example.com", body=body, request=request)
self.assertIsNone(spider._get_sitemap_body(response))