Scrapy, a fast high-level web crawling & scraping framework for Python.
Require start_queue_cls.
Andrey Rakhmatullinf24bc749ea5b
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 1b7fb652d..78329460e 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -5,16 +5,13 @@ import loggingfrom abc import abstractmethodfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any-from warnings import warn# working around https://github.com/sphinx-doc/sphinx/issues/10400from twisted.internet.defer import Deferred # noqa: TC002-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.spiders import Spider # noqa: TC001from scrapy.utils.job import job_dirfrom scrapy.utils.misc import build_from_crawler, load_object-from scrapy.utils.python import global_object_nameif TYPE_CHECKING:# requires queuelib >= 1.6.2@@ -450,28 +447,13 @@ class Scheduler(BaseScheduler):"""Create a new priority queue instance, with in-memory storage"""assert self.crawlerassert self.pqclass- try:- return build_from_crawler(- self.pqclass,- self.crawler,- downstream_queue_cls=self.mqclass,- key="",- start_queue_cls=self._smqclass,- )- except TypeError: # pragma: no cover- warn(- f"The __init__ method of {global_object_name(self.pqclass)} "- "does not support a `start_queue_cls` keyword-only "- "parameter.",- ScrapyDeprecationWarning,- stacklevel=2,- )- return build_from_crawler(- self.pqclass,- self.crawler,- downstream_queue_cls=self.mqclass,- key="",- )+ return build_from_crawler(+ self.pqclass,+ self.crawler,+ downstream_queue_cls=self.mqclass,+ key="",+ start_queue_cls=self._smqclass,+ )def _dq(self) -> ScrapyPriorityQueue:"""Create a new priority queue instance, with disk storage"""@@ -479,30 +461,14 @@ class Scheduler(BaseScheduler):assert self.dqdirassert self.pqclassstate = self._read_dqs_state(self.dqdir)- try:- q = build_from_crawler(- self.pqclass,- self.crawler,- downstream_queue_cls=self.dqclass,- key=self.dqdir,- startprios=state,- start_queue_cls=self._sdqclass,- )- except TypeError: # pragma: no cover- warn(- f"The __init__ method of {global_object_name(self.pqclass)} "- "does not support a `start_queue_cls` keyword-only "- "parameter.",- ScrapyDeprecationWarning,- stacklevel=2,- )- q = build_from_crawler(- self.pqclass,- self.crawler,- downstream_queue_cls=self.dqclass,- key=self.dqdir,- startprios=state,- )+ q = build_from_crawler(+ self.pqclass,+ self.crawler,+ downstream_queue_cls=self.dqclass,+ key=self.dqdir,+ startprios=state,+ start_queue_cls=self._sdqclass,+ )if q:logger.info("Resuming crawl (%(queuesize)d requests scheduled)",