Scrapy, a fast high-level web crawling & scraping framework for Python.
add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider extension
guillermo-bondonno0b8604bb5d8b
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex 4627e7f98..6ebf98e65 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dictfrom scrapy import Request, Spider, signalsfrom scrapy.exceptions import NotConfigured+from scrapy.signalmanager import dispatcherif TYPE_CHECKING:from twisted.python.failure import Failure@@ -36,6 +37,9 @@ class CloseSpider:"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),"timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),+ "pagecount_no_item": crawler.settings.getint(+ "CLOSESPIDER_PAGECOUNT_NO_ITEM"+ ),}if not any(self.close_on.values()):@@ -60,6 +64,19 @@ class CloseSpider:crawler.signals.connect(self.item_scraped_no_item, signal=signals.item_scraped)+ if self.close_on.get("pagecount_no_item"):+ if self.page_count not in dispatcher.getReceivers(+ signal=signals.response_received+ ):+ crawler.signals.connect(+ self.page_count, signal=signals.response_received+ )++ if self.item_scraped not in dispatcher.getReceivers(+ signal=signals.item_scraped+ ):+ crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)+crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)@classmethod@@ -74,9 +91,16 @@ class CloseSpider:def page_count(self, response: Response, request: Request, spider: Spider) -> None:self.counter["pagecount"] += 1+ self.counter["pagecount_since_last_item"] += 1if self.counter["pagecount"] == self.close_on["pagecount"]:assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_pagecount")+ if self.close_on["pagecount_no_item"] and (+ self.counter["pagecount_since_last_item"]+ >= self.close_on["pagecount_no_item"]+ ):+ assert self.crawler.engine+ self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")def spider_opened(self, spider: Spider) -> None:from twisted.internet import reactor@@ -91,6 +115,7 @@ class CloseSpider:def item_scraped(self, item: Any, spider: Spider) -> None:self.counter["itemcount"] += 1+ self.counter["pagecount_since_last_item"] = 0if self.counter["itemcount"] == self.close_on["itemcount"]:assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_itemcount")diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pemnew file mode 100644index 000000000..c10121fbf--- /dev/null+++ b/tests/keys/mitmproxy-dhparam.pem@@ -0,0 +1,14 @@++-----BEGIN DH PARAMETERS-----+MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3+O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv+j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ+Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB+chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC+ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq+o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX+IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv+A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8+6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I+rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=+-----END DH PARAMETERS-----diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 38ede70e4..caaa9f183 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -34,6 +34,19 @@ class TestCloseSpider(TestCase):pagecount = crawler.stats.get_value("response_received_count")self.assertTrue(pagecount >= close_on)+ @defer.inlineCallbacks+ def test_closespider_pagecount_no_item(self):+ close_on = 5+ crawler = get_crawler(+ FollowAllSpider,+ {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on},+ )+ yield crawler.crawl(mockserver=self.mockserver)+ reason = crawler.spider.meta["close_reason"]+ self.assertEqual(reason, "closespider_pagecount_no_item")+ pagecount = crawler.stats.get_value("response_received_count")+ self.assertTrue(pagecount >= close_on)+@defer.inlineCallbacksdef test_closespider_errorcount(self):close_on = 5