Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider extension

guillermo-bondonno0b8604bb5d8b
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 4627e7f98..6ebf98e65 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured
+from scrapy.signalmanager import dispatcher
if TYPE_CHECKING:
from twisted.python.failure import Failure
@@ -36,6 +37,9 @@ class CloseSpider:
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
"timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
+ "pagecount_no_item": crawler.settings.getint(
+ "CLOSESPIDER_PAGECOUNT_NO_ITEM"
+ ),
}
if not any(self.close_on.values()):
@@ -60,6 +64,19 @@ class CloseSpider:
crawler.signals.connect(
self.item_scraped_no_item, signal=signals.item_scraped
)
+ if self.close_on.get("pagecount_no_item"):
+ if self.page_count not in dispatcher.getReceivers(
+ signal=signals.response_received
+ ):
+ crawler.signals.connect(
+ self.page_count, signal=signals.response_received
+ )
+
+ if self.item_scraped not in dispatcher.getReceivers(
+ signal=signals.item_scraped
+ ):
+ crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
@@ -74,9 +91,16 @@ class CloseSpider:
def page_count(self, response: Response, request: Request, spider: Spider) -> None:
self.counter["pagecount"] += 1
+ self.counter["pagecount_since_last_item"] += 1
if self.counter["pagecount"] == self.close_on["pagecount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_pagecount")
+ if self.close_on["pagecount_no_item"] and (
+ self.counter["pagecount_since_last_item"]
+ >= self.close_on["pagecount_no_item"]
+ ):
+ assert self.crawler.engine
+ self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")
def spider_opened(self, spider: Spider) -> None:
from twisted.internet import reactor
@@ -91,6 +115,7 @@ class CloseSpider:
def item_scraped(self, item: Any, spider: Spider) -> None:
self.counter["itemcount"] += 1
+ self.counter["pagecount_since_last_item"] = 0
if self.counter["itemcount"] == self.close_on["itemcount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_itemcount")
diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem
new file mode 100644
index 000000000..c10121fbf
--- /dev/null
+++ b/tests/keys/mitmproxy-dhparam.pem
@@ -0,0 +1,14 @@
+
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3
+O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv
+j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ
+Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB
+chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC
+ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq
+o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX
+IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv
+A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8
+6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I
+rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
+-----END DH PARAMETERS-----
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 38ede70e4..caaa9f183 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -34,6 +34,19 @@ class TestCloseSpider(TestCase):
pagecount = crawler.stats.get_value("response_received_count")
self.assertTrue(pagecount >= close_on)
+ @defer.inlineCallbacks
+ def test_closespider_pagecount_no_item(self):
+ close_on = 5
+ crawler = get_crawler(
+ FollowAllSpider,
+ {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on},
+ )
+ yield crawler.crawl(mockserver=self.mockserver)
+ reason = crawler.spider.meta["close_reason"]
+ self.assertEqual(reason, "closespider_pagecount_no_item")
+ pagecount = crawler.stats.get_value("response_received_count")
+ self.assertTrue(pagecount >= close_on)
+
@defer.inlineCallbacks
def test_closespider_errorcount(self):
close_on = 5