Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fixes for queuelib.

Andrey Rakhmatullinbd0d4cee8857
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index ab59c0d14..f30a5d9c9 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -6,7 +6,6 @@ from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Type, cast
-from queuelib.queue import BaseQueue
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
@@ -19,6 +18,9 @@ from scrapy.utils.job import job_dir
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
+ # requires queuelib >= 1.6.2
+ from queuelib.queue import BaseQueue
+
# typing.Self requires Python 3.11
from typing_extensions import Self
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 4676b058e..6f80ee388 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
- class DirectoriesCreated(queue_class):
+ class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]
def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
dirname = Path(path).parent
if not dirname.exists():
@@ -37,7 +37,7 @@ def _serializable_queue(
serialize: Callable[[Any], bytes],
deserialize: Callable[[bytes], Any],
) -> Type[queue.BaseQueue]:
- class SerializableQueue(queue_class):
+ class SerializableQueue(queue_class): # type: ignore[valid-type,misc]
def push(self, obj: Any) -> None:
s = serialize(obj)
super().push(s)
@@ -71,7 +71,7 @@ def _serializable_queue(
def _scrapy_serialization_queue(
queue_class: Type[queue.BaseQueue],
) -> Type[queue.BaseQueue]:
- class ScrapyRequestQueue(queue_class):
+ class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]
def __init__(self, crawler: Crawler, key: str):
self.spider = crawler.spider
super().__init__(key)
@@ -110,7 +110,7 @@ def _scrapy_serialization_queue(
def _scrapy_non_serialization_queue(
queue_class: Type[queue.BaseQueue],
) -> Type[queue.BaseQueue]:
- class ScrapyRequestQueue(queue_class):
+ class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
return cls()
@@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes:
raise ValueError(str(e)) from e
+# queue.*Queue aren't subclasses of queue.BaseQueue
_PickleFifoSerializationDiskQueue = _serializable_queue(
- _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads
+ _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type]
)
_PickleLifoSerializationDiskQueue = _serializable_queue(
- _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads
+ _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type]
)
_MarshalFifoSerializationDiskQueue = _serializable_queue(
- _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads
+ _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type]
)
_MarshalLifoSerializationDiskQueue = _serializable_queue(
- _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads
+ _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type]
)
# public queue classes
@@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu
PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
-FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
-LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
+FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type]
+LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type]