Scrapy, a fast high-level web crawling & scraping framework for Python.
Change name, log once
Darshan Chaudhary472a8a47d0b2
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rstindex 303076015..093cf9bb5 100644--- a/docs/topics/jobs.rst+++ b/docs/topics/jobs.rst@@ -96,4 +96,6 @@ But this will::somearg = response.meta['somearg']print "the argument passed is:", somearg+If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.+.. _pickle: http://docs.python.org/library/pickle.htmldiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex c845c59b9..d03da44c6 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1026,6 +1026,16 @@ Default: ``'scrapy.core.scheduler.Scheduler'``The scheduler to use for crawling.+.. setting:: SCHEDULER_DEBUG++SCHEDULER_DEBUG+---------------++Default: ``False``++Setting to ``True`` will log the first unserializable request encountered.+Stats collected.+.. setting:: SPIDER_CONTRACTSSPIDER_CONTRACTSdiff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 8be9a0019..e0788a8c5 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -30,7 +30,7 @@ class Scheduler(object):pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])- logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')+ logunser = settings.getbool('SCHEDULER_DEBUG')return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)@@ -86,9 +86,13 @@ class Scheduler(object):self.dqs.push(reqd, -request.priority)except ValueError as e: # non serializable requestif self.logunser:- logger.error("Unable to serialize request: %(request)s - reason: %(reason)s",- {'request': request, 'reason': e},+ msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"+ " - no more unserializable requests will be logged"+ " (stats being collected)")+ logger.error(msg, {'request': request, 'reason': e},exc_info=True, extra={'spider': self.spider})+ self.logunser = False+ self.stats.inc_value('scheduler/unserializable', spider=self.spider)returnelse:return Truediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 2c267b4cc..673df5129 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -191,7 +191,7 @@ LOG_STDOUT = FalseLOG_LEVEL = 'DEBUG'LOG_FILE = None-LOG_UNSERIALIZABLE_REQUESTS = False+SCHEDULER_DEBUG = FalseLOGSTATS_INTERVAL = 60.0