Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow core Scheduler priority queue customization

nyov2a6524ee3a21
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index d132bbfd3..8be9a0019 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -3,7 +3,6 @@ import json
import logging
from os.path import join, exists
-from queuelib import PriorityQueue
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy.utils.misc import load_object
from scrapy.utils.job import job_dir
@@ -13,9 +12,11 @@ logger = logging.getLogger(__name__)
class Scheduler(object):
- def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None):
+ def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
+ logunser=False, stats=None, pqclass=None):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
+ self.pqclass = pqclass
self.dqclass = dqclass
self.mqclass = mqclass
self.logunser = logunser
@@ -26,17 +27,19 @@ class Scheduler(object):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
dupefilter = dupefilter_cls.from_settings(settings)
+ pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
- return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats)
+ return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
+ stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
def has_pending_requests(self):
return len(self) > 0
def open(self, spider):
self.spider = spider
- self.mqs = PriorityQueue(self._newmq)
+ self.mqs = self.pqclass(self._newmq)
self.dqs = self._dq() if self.dqdir else None
return self.df.open()
@@ -112,7 +115,7 @@ class Scheduler(object):
prios = json.load(f)
else:
prios = ()
- q = PriorityQueue(self._newdq, startprios=prios)
+ q = self.pqclass(self._newdq, startprios=prios)
if q:
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
{'queuesize': len(q)}, extra={'spider': self.spider})
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8e6d29045..d449c4891 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,6 +234,7 @@ ROBOTSTXT_OBEY = False
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
+SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'