Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

introduce CLOSESPIDER_TIMEOUT_NO_ITEM in CloseSpider

Kevin Lloyd Bernal3f5bbe3a8fab
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 96e0216b8..8d4749ab3 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following
settings:
* :setting:`CLOSESPIDER_TIMEOUT`
+* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
* :setting:`CLOSESPIDER_ITEMCOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
@@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
timeout.
+.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
+
+CLOSESPIDER_TIMEOUT_NO_ITEM
+"""""""""""""""""""""""""""
+
+Default: ``0``
+
+An integer which specifies a number of seconds. If the spider has not produced
+any items in the last number of seconds, it will be closed with the reason
+``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed
+regardless if it hasn't produced any items.
+
.. setting:: CLOSESPIDER_ITEMCOUNT
CLOSESPIDER_ITEMCOUNT
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index bb6f832f2..456470efd 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -4,11 +4,14 @@ conditions are met.
See documentation in docs/topics/extensions.rst
"""
+import logging
from collections import defaultdict
from scrapy import signals
from scrapy.exceptions import NotConfigured
+logger = logging.getLogger(__name__)
+
class CloseSpider:
def __init__(self, crawler):
@@ -19,6 +22,7 @@ class CloseSpider:
"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
+ "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
}
if not any(self.close_on.values()):
@@ -34,6 +38,15 @@ class CloseSpider:
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
if self.close_on.get("itemcount"):
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+ if self.close_on.get("timeout_no_item"):
+ self.timeout_no_item = self.close_on["timeout_no_item"]
+ self.items_in_period = 0
+ crawler.signals.connect(
+ self.spider_opened_no_item, signal=signals.spider_opened
+ )
+ crawler.signals.connect(
+ self.item_scraped_no_item, signal=signals.item_scraped
+ )
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
@@ -69,3 +82,31 @@ class CloseSpider:
task = getattr(self, "task", False)
if task and task.active():
task.cancel()
+
+ task_no_item = getattr(self, "task_no_item", False)
+ if task_no_item.running:
+ task_no_item.stop()
+
+ def spider_opened_no_item(self, spider):
+ from twisted.internet import task
+
+ self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
+ self.task_no_item.start(self.timeout_no_item, now=False)
+
+ logger.info(
+ f"Spider will stop when no items are produced after "
+ f"{self.timeout_no_item} seconds."
+ )
+
+ def item_scraped_no_item(self, item, spider):
+ self.items_in_period += 1
+
+ def _count_items_produced(self, spider):
+ if self.items_in_period >= 1:
+ self.items_in_period = 0
+ else:
+ logger.info(
+ f"Closing spider since no items were produced in the last "
+ f"{self.timeout_no_item} seconds."
+ )
+ self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 9b39187d5..259a1a4ad 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider
+from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider
class TestCloseSpider(TestCase):
@@ -54,3 +54,13 @@ class TestCloseSpider(TestCase):
self.assertEqual(reason, "closespider_timeout")
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
self.assertTrue(total_seconds >= close_on)
+
+ @defer.inlineCallbacks
+ def test_closespider_timeout_no_item(self):
+ timeout = 1
+ crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
+ yield crawler.crawl(n=3, total=10, mockserver=self.mockserver)
+ reason = crawler.spider.meta["close_reason"]
+ self.assertEqual(reason, "closespider_timeout_no_item")
+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")
+ self.assertTrue(total_seconds >= timeout)