Scrapy, a fast high-level web crawling & scraping framework for Python.
update docs for settings
darshanime0c77b6d033a0
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rstindex 093cf9bb5..55105dcfd 100644--- a/docs/topics/jobs.rst+++ b/docs/topics/jobs.rst@@ -96,6 +96,8 @@ But this will::somearg = response.meta['somearg']print "the argument passed is:", somearg-If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.+If you wish to log the requests that couldn't be serialized, you can set the+``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page.+It is ``False`` by default... _pickle: http://docs.python.org/library/pickle.htmldiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex d03da44c6..d14f230e4 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1033,8 +1033,15 @@ SCHEDULER_DEBUGDefault: ``False``-Setting to ``True`` will log the first unserializable request encountered.-Stats collected.+Setting to ``True`` will log debug information about the requests scheduler.+This currently logs(only once) if the requests cannot be serialized to disk.+Stats counter (``scheduler/unserializable``) tracks the number of times this happens.++Example entry in logs::++ 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: <request>+ - reason: <exception> - no more unserializable requests will be logged (stats being+ collected).. setting:: SPIDER_CONTRACTSdiff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex a3cb1bcff..dcd6fb989 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -84,15 +84,16 @@ class Scheduler(object):try:reqd = request_to_dict(request, self.spider)self.dqs.push(reqd, -request.priority)- except ValueError as e: # non serializable request+ except ValueError as e: # non serializable requestif self.logunser:- msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"- " - no more unserializable requests will be logged"- " (stats being collected)")+ msg = ("Unable to serialize request: %(request)s - reason:"+ " %(reason)s - no more unserializable requests will be"+ " logged (stats being collected)")logger.error(msg, {'request': request, 'reason': e},exc_info=True, extra={'spider': self.spider})self.logunser = False- self.stats.inc_value('scheduler/unserializable', spider=self.spider)+ self.stats.inc_value('scheduler/unserializable',+ spider=self.spider)returnelse:return True