Scrapy, a fast high-level web crawling & scraping framework for Python.
CloseSpider can be raised on spider_idle signal handler
Iván de Prado Alonso812b4bb51855
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex 3d838fb63..ee9d95836 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -268,6 +268,9 @@ spider_idleYou may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception toprevent the spider from being closed.+ Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`+ exception to provide a custom spider closing reason.+This signal does not support returning deferreds from its handlers.:param spider: the spider which has gone idlediff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex dd3225082..949d89e76 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -15,7 +15,8 @@ from twisted.python.failure import Failurefrom scrapy import signalsfrom scrapy.core.scraper import Scraper-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \+ CloseSpiderfrom scrapy.http import Response, Requestfrom scrapy.settings import BaseSettingsfrom scrapy.spiders import Spider@@ -325,14 +326,19 @@ class ExecutionEngine:Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpiderexception, the spider is not closed until the next loop and this function is guaranteed to be called- (at least) once again.+ (at least) once again. A handler can raise CloseSpider to provide a custom closing reason"""assert self.spider is not None # typing- res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)- if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):+ expected_ex = (DontCloseSpider, CloseSpider)+ res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)+ detected_ex = {ex: x.value+ for _, x in res for ex in expected_ex+ if isinstance(x, Failure) and isinstance(x.value, ex)}+ if DontCloseSpider in detected_ex:return Noneif self.spider_is_idle():- self.close_spider(self.spider, reason='finished')+ reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished'+ self.close_spider(self.spider, reason=reason)def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:"""Close (cancel) spider and clear all its outstanding requests"""diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex 115707182..62808f3ce 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -1,5 +1,5 @@"""Helper functions for working with signals"""-+import collectionsimport loggingfrom twisted.internet.defer import DeferredList, Deferred@@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_infologger = logging.getLogger(__name__)-class _IgnoredException(Exception):- pass--def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):"""Like pydispatcher.robust.sendRobust but it also logs errors and returnsFailures instead of exceptions."""- dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)+ dont_log = named.pop('dont_log', ())+ dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)+ dont_log += (StopDownload, )spider = named.get('spider', None)responses = []for receiver in liveReceivers(getAllReceivers(sender, signal)):