Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Full typing for scrapy/squeues.py.

Andrey Rakhmatullin5f7fd2a65340
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index e20f60f06..4676b058e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -2,20 +2,28 @@
Scheduler queues
"""
+from __future__ import annotations
+
import marshal
import pickle # nosec
from os import PathLike
from pathlib import Path
-from typing import Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
from queuelib import queue
+from scrapy import Request
+from scrapy.crawler import Crawler
from scrapy.utils.request import request_from_dict
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
-def _with_mkdir(queue_class):
+def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
class DirectoriesCreated(queue_class):
- def __init__(self, path: Union[str, PathLike], *args, **kwargs):
+ def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
dirname = Path(path).parent
if not dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)
@@ -24,18 +32,23 @@ def _with_mkdir(queue_class):
return DirectoriesCreated
-def _serializable_queue(queue_class, serialize, deserialize):
+def _serializable_queue(
+ queue_class: Type[queue.BaseQueue],
+ serialize: Callable[[Any], bytes],
+ deserialize: Callable[[bytes], Any],
+) -> Type[queue.BaseQueue]:
class SerializableQueue(queue_class):
- def push(self, obj):
+ def push(self, obj: Any) -> None:
s = serialize(obj)
super().push(s)
- def pop(self):
+ def pop(self) -> Optional[Any]:
s = super().pop()
if s:
return deserialize(s)
+ return None
- def peek(self):
+ def peek(self) -> Optional[Any]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize):
) from ex
if s:
return deserialize(s)
+ return None
return SerializableQueue
-def _scrapy_serialization_queue(queue_class):
+def _scrapy_serialization_queue(
+ queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
class ScrapyRequestQueue(queue_class):
- def __init__(self, crawler, key):
+ def __init__(self, crawler: Crawler, key: str):
self.spider = crawler.spider
super().__init__(key)
@classmethod
- def from_crawler(cls, crawler, key, *args, **kwargs):
+ def from_crawler(
+ cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any
+ ) -> Self:
return cls(crawler, key)
- def push(self, request):
- request = request.to_dict(spider=self.spider)
- return super().push(request)
+ def push(self, request: Request) -> None:
+ request_dict = request.to_dict(spider=self.spider)
+ super().push(request_dict)
- def pop(self):
+ def pop(self) -> Optional[Request]:
request = super().pop()
if not request:
return None
return request_from_dict(request, spider=self.spider)
- def peek(self):
+ def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class):
return ScrapyRequestQueue
-def _scrapy_non_serialization_queue(queue_class):
+def _scrapy_non_serialization_queue(
+ queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
class ScrapyRequestQueue(queue_class):
@classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
return cls()
- def peek(self):
+ def peek(self) -> Optional[Any]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class):
return ScrapyRequestQueue
-def _pickle_serialize(obj):
+def _pickle_serialize(obj: Any) -> bytes:
try:
return pickle.dumps(obj, protocol=4)
# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)