Scrapy, a fast high-level web crawling & scraping framework for Python.
introduce CLOSESPIDER_TIMEOUT_NO_ITEM in CloseSpider
Kevin Lloyd Bernal3f5bbe3a8fab
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rstindex 96e0216b8..8d4749ab3 100644--- a/docs/topics/extensions.rst+++ b/docs/topics/extensions.rst@@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the followingsettings:* :setting:`CLOSESPIDER_TIMEOUT`+* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`* :setting:`CLOSESPIDER_ITEMCOUNT`* :setting:`CLOSESPIDER_PAGECOUNT`* :setting:`CLOSESPIDER_ERRORCOUNT`@@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with thereason ``closespider_timeout``. If zero (or non set), spiders won't be closed bytimeout.+.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM++CLOSESPIDER_TIMEOUT_NO_ITEM+"""""""""""""""""""""""""""++Default: ``0``++An integer which specifies a number of seconds. If the spider has not produced+any items in the last number of seconds, it will be closed with the reason+``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed+regardless if it hasn't produced any items.+.. setting:: CLOSESPIDER_ITEMCOUNTCLOSESPIDER_ITEMCOUNTdiff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex bb6f832f2..456470efd 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -4,11 +4,14 @@ conditions are met.See documentation in docs/topics/extensions.rst"""+import loggingfrom collections import defaultdictfrom scrapy import signalsfrom scrapy.exceptions import NotConfigured+logger = logging.getLogger(__name__)+class CloseSpider:def __init__(self, crawler):@@ -19,6 +22,7 @@ class CloseSpider:"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),+ "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),}if not any(self.close_on.values()):@@ -34,6 +38,15 @@ class CloseSpider:crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)if self.close_on.get("itemcount"):crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)+ if self.close_on.get("timeout_no_item"):+ self.timeout_no_item = self.close_on["timeout_no_item"]+ self.items_in_period = 0+ crawler.signals.connect(+ self.spider_opened_no_item, signal=signals.spider_opened+ )+ crawler.signals.connect(+ self.item_scraped_no_item, signal=signals.item_scraped+ )crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)@classmethod@@ -69,3 +82,31 @@ class CloseSpider:task = getattr(self, "task", False)if task and task.active():task.cancel()++ task_no_item = getattr(self, "task_no_item", False)+ if task_no_item.running:+ task_no_item.stop()++ def spider_opened_no_item(self, spider):+ from twisted.internet import task++ self.task_no_item = task.LoopingCall(self._count_items_produced, spider)+ self.task_no_item.start(self.timeout_no_item, now=False)++ logger.info(+ f"Spider will stop when no items are produced after "+ f"{self.timeout_no_item} seconds."+ )++ def item_scraped_no_item(self, item, spider):+ self.items_in_period += 1++ def _count_items_produced(self, spider):+ if self.items_in_period >= 1:+ self.items_in_period = 0+ else:+ logger.info(+ f"Closing spider since no items were produced in the last "+ f"{self.timeout_no_item} seconds."+ )+ self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 9b39187d5..259a1a4ad 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCasefrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider+from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpiderclass TestCloseSpider(TestCase):@@ -54,3 +54,13 @@ class TestCloseSpider(TestCase):self.assertEqual(reason, "closespider_timeout")total_seconds = crawler.stats.get_value("elapsed_time_seconds")self.assertTrue(total_seconds >= close_on)++ @defer.inlineCallbacks+ def test_closespider_timeout_no_item(self):+ timeout = 1+ crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})+ yield crawler.crawl(n=3, total=10, mockserver=self.mockserver)+ reason = crawler.spider.meta["close_reason"]+ self.assertEqual(reason, "closespider_timeout_no_item")+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")+ self.assertTrue(total_seconds >= timeout)