Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/squeues.py.
Andrey Rakhmatullin5f7fd2a65340
diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex e20f60f06..4676b058e 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -2,20 +2,28 @@Scheduler queues"""+from __future__ import annotations+import marshalimport pickle # nosecfrom os import PathLikefrom pathlib import Path-from typing import Union+from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Unionfrom queuelib import queue+from scrapy import Request+from scrapy.crawler import Crawlerfrom scrapy.utils.request import request_from_dict+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+-def _with_mkdir(queue_class):+def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:class DirectoriesCreated(queue_class):- def __init__(self, path: Union[str, PathLike], *args, **kwargs):+ def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):dirname = Path(path).parentif not dirname.exists():dirname.mkdir(parents=True, exist_ok=True)@@ -24,18 +32,23 @@ def _with_mkdir(queue_class):return DirectoriesCreated-def _serializable_queue(queue_class, serialize, deserialize):+def _serializable_queue(+ queue_class: Type[queue.BaseQueue],+ serialize: Callable[[Any], bytes],+ deserialize: Callable[[bytes], Any],+) -> Type[queue.BaseQueue]:class SerializableQueue(queue_class):- def push(self, obj):+ def push(self, obj: Any) -> None:s = serialize(obj)super().push(s)- def pop(self):+ def pop(self) -> Optional[Any]:s = super().pop()if s:return deserialize(s)+ return None- def peek(self):+ def peek(self) -> Optional[Any]:"""Returns the next object to be returned by :meth:`pop`,but without removing it from the queue.@@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize):) from exif s:return deserialize(s)+ return Nonereturn SerializableQueue-def _scrapy_serialization_queue(queue_class):+def _scrapy_serialization_queue(+ queue_class: Type[queue.BaseQueue],+) -> Type[queue.BaseQueue]:class ScrapyRequestQueue(queue_class):- def __init__(self, crawler, key):+ def __init__(self, crawler: Crawler, key: str):self.spider = crawler.spidersuper().__init__(key)@classmethod- def from_crawler(cls, crawler, key, *args, **kwargs):+ def from_crawler(+ cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any+ ) -> Self:return cls(crawler, key)- def push(self, request):- request = request.to_dict(spider=self.spider)- return super().push(request)+ def push(self, request: Request) -> None:+ request_dict = request.to_dict(spider=self.spider)+ super().push(request_dict)- def pop(self):+ def pop(self) -> Optional[Request]:request = super().pop()if not request:return Nonereturn request_from_dict(request, spider=self.spider)- def peek(self):+ def peek(self) -> Optional[Request]:"""Returns the next object to be returned by :meth:`pop`,but without removing it from the queue.@@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class):return ScrapyRequestQueue-def _scrapy_non_serialization_queue(queue_class):+def _scrapy_non_serialization_queue(+ queue_class: Type[queue.BaseQueue],+) -> Type[queue.BaseQueue]:class ScrapyRequestQueue(queue_class):@classmethod- def from_crawler(cls, crawler, *args, **kwargs):+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:return cls()- def peek(self):+ def peek(self) -> Optional[Any]:"""Returns the next object to be returned by :meth:`pop`,but without removing it from the queue.@@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class):return ScrapyRequestQueue-def _pickle_serialize(obj):+def _pickle_serialize(obj: Any) -> bytes:try:return pickle.dumps(obj, protocol=4)# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)