Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow core Scheduler priority queue customization
nyov2a6524ee3a21
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex d132bbfd3..8be9a0019 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -3,7 +3,6 @@ import jsonimport loggingfrom os.path import join, exists-from queuelib import PriorityQueuefrom scrapy.utils.reqser import request_to_dict, request_from_dictfrom scrapy.utils.misc import load_objectfrom scrapy.utils.job import job_dir@@ -13,9 +12,11 @@ logger = logging.getLogger(__name__)class Scheduler(object):- def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None):+ def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,+ logunser=False, stats=None, pqclass=None):self.df = dupefilterself.dqdir = self._dqdir(jobdir)+ self.pqclass = pqclassself.dqclass = dqclassself.mqclass = mqclassself.logunser = logunser@@ -26,17 +27,19 @@ class Scheduler(object):settings = crawler.settingsdupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])dupefilter = dupefilter_cls.from_settings(settings)+ pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')- return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats)+ return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,+ stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)def has_pending_requests(self):return len(self) > 0def open(self, spider):self.spider = spider- self.mqs = PriorityQueue(self._newmq)+ self.mqs = self.pqclass(self._newmq)self.dqs = self._dq() if self.dqdir else Nonereturn self.df.open()@@ -112,7 +115,7 @@ class Scheduler(object):prios = json.load(f)else:prios = ()- q = PriorityQueue(self._newdq, startprios=prios)+ q = self.pqclass(self._newdq, startprios=prios)if q:logger.info("Resuming crawl (%(queuesize)d requests scheduled)",{'queuesize': len(q)}, extra={'spider': self.spider})diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 8e6d29045..d449c4891 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -234,6 +234,7 @@ ROBOTSTXT_OBEY = FalseSCHEDULER = 'scrapy.core.scheduler.Scheduler'SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'+SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'