Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CloseSpider can be raised on spider_idle signal handler

Iván de Prado Alonso812b4bb51855
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3d838fb63..ee9d95836 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -268,6 +268,9 @@ spider_idle
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
+ Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
+ exception to provide a custom spider closing reason.
+
This signal does not support returning deferreds from its handlers.
:param spider: the spider which has gone idle
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index dd3225082..949d89e76 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -15,7 +15,8 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \
+ CloseSpider
from scrapy.http import Response, Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
@@ -325,14 +326,19 @@ class ExecutionEngine:
Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
exception, the spider is not closed until the next loop and this function is guaranteed to be called
- (at least) once again.
+ (at least) once again. A handler can raise CloseSpider to provide a custom closing reason
"""
assert self.spider is not None # typing
- res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
- if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
+ expected_ex = (DontCloseSpider, CloseSpider)
+ res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
+ detected_ex = {ex: x.value
+ for _, x in res for ex in expected_ex
+ if isinstance(x, Failure) and isinstance(x.value, ex)}
+ if DontCloseSpider in detected_ex:
return None
if self.spider_is_idle():
- self.close_spider(self.spider, reason='finished')
+ reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished'
+ self.close_spider(self.spider, reason=reason)
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
"""Close (cancel) spider and clear all its outstanding requests"""
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 115707182..62808f3ce 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,5 +1,5 @@
"""Helper functions for working with signals"""
-
+import collections
import logging
from twisted.internet.defer import DeferredList, Deferred
@@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
-class _IgnoredException(Exception):
- pass
-
-
def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
- dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
+ dont_log = named.pop('dont_log', ())
+ dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+ dont_log += (StopDownload, )
spider = named.get('spider', None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):