Scrapy, a fast high-level web crawling & scraping framework for Python.
Fixes for queuelib.
Andrey Rakhmatullinbd0d4cee8857
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex ab59c0d14..f30a5d9c9 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -6,7 +6,6 @@ from abc import abstractmethodfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Optional, Type, cast-from queuelib.queue import BaseQueuefrom twisted.internet.defer import Deferredfrom scrapy.crawler import Crawler@@ -19,6 +18,9 @@ from scrapy.utils.job import job_dirfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:+ # requires queuelib >= 1.6.2+ from queuelib.queue import BaseQueue+# typing.Self requires Python 3.11from typing_extensions import Selfdiff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 4676b058e..6f80ee388 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -22,7 +22,7 @@ if TYPE_CHECKING:def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:- class DirectoriesCreated(queue_class):+ class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):dirname = Path(path).parentif not dirname.exists():@@ -37,7 +37,7 @@ def _serializable_queue(serialize: Callable[[Any], bytes],deserialize: Callable[[bytes], Any],) -> Type[queue.BaseQueue]:- class SerializableQueue(queue_class):+ class SerializableQueue(queue_class): # type: ignore[valid-type,misc]def push(self, obj: Any) -> None:s = serialize(obj)super().push(s)@@ -71,7 +71,7 @@ def _serializable_queue(def _scrapy_serialization_queue(queue_class: Type[queue.BaseQueue],) -> Type[queue.BaseQueue]:- class ScrapyRequestQueue(queue_class):+ class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]def __init__(self, crawler: Crawler, key: str):self.spider = crawler.spidersuper().__init__(key)@@ -110,7 +110,7 @@ def _scrapy_serialization_queue(def _scrapy_non_serialization_queue(queue_class: Type[queue.BaseQueue],) -> Type[queue.BaseQueue]:- class ScrapyRequestQueue(queue_class):+ class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]@classmethoddef from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:return cls()@@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes:raise ValueError(str(e)) from e+# queue.*Queue aren't subclasses of queue.BaseQueue_PickleFifoSerializationDiskQueue = _serializable_queue(- _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads+ _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type])_PickleLifoSerializationDiskQueue = _serializable_queue(- _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads+ _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type])_MarshalFifoSerializationDiskQueue = _serializable_queue(- _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads+ _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type])_MarshalLifoSerializationDiskQueue = _serializable_queue(- _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads+ _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type])# public queue classes@@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQuPickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)-FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)-LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)+FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type]+LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type]