Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

update docs for settings

darshanime0c77b6d033a0
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 093cf9bb5..55105dcfd 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -96,6 +96,8 @@ But this will::
somearg = response.meta['somearg']
print "the argument passed is:", somearg
-If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.
+If you wish to log the requests that couldn't be serialized, you can set the
+``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page.
+It is ``False`` by default.
.. _pickle: http://docs.python.org/library/pickle.html
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d03da44c6..d14f230e4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1033,8 +1033,15 @@ SCHEDULER_DEBUG
Default: ``False``
-Setting to ``True`` will log the first unserializable request encountered.
-Stats collected.
+Setting to ``True`` will log debug information about the requests scheduler.
+This currently logs(only once) if the requests cannot be serialized to disk.
+Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
+
+Example entry in logs::
+
+ 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: <request>
+ - reason: <exception> - no more unserializable requests will be logged (stats being
+ collected)
.. setting:: SPIDER_CONTRACTS
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a3cb1bcff..dcd6fb989 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -84,15 +84,16 @@ class Scheduler(object):
try:
reqd = request_to_dict(request, self.spider)
self.dqs.push(reqd, -request.priority)
- except ValueError as e: # non serializable request
+ except ValueError as e: # non serializable request
if self.logunser:
- msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"
- " - no more unserializable requests will be logged"
- " (stats being collected)")
+ msg = ("Unable to serialize request: %(request)s - reason:"
+ " %(reason)s - no more unserializable requests will be"
+ " logged (stats being collected)")
logger.error(msg, {'request': request, 'reason': e},
exc_info=True, extra={'spider': self.spider})
self.logunser = False
- self.stats.inc_value('scheduler/unserializable', spider=self.spider)
+ self.stats.inc_value('scheduler/unserializable',
+ spider=self.spider)
return
else:
return True