Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/pqueues.py.
Andrey Rakhmatullin0c8e21b8acfc
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex ecd3e8b56..f88da41ea 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -143,7 +143,7 @@ class Downloader:return key, self.slots[key]- def _get_slot_key(self, request: Request, spider: Spider) -> str:+ def _get_slot_key(self, request: Request, spider: Any) -> str:if self.DOWNLOAD_SLOT in request.meta:return cast(str, request.meta[self.DOWNLOAD_SLOT])diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 593667f1f..213ad590d 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -1,12 +1,32 @@+from __future__ import annotations+import hashlibimport logging-+from typing import (+ TYPE_CHECKING,+ Dict,+ Iterable,+ List,+ Optional,+ Protocol,+ Tuple,+ Type,+ cast,+)++from scrapy import Request+from scrapy.core.downloader import Downloader+from scrapy.crawler import Crawlerfrom scrapy.utils.misc import build_from_crawler+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+logger = logging.getLogger(__name__)-def _path_safe(text):+def _path_safe(text: str) -> str:"""Return a filesystem-safe version of a string ``text``@@ -24,6 +44,18 @@ def _path_safe(text):return "-".join([pathable_slot, unique_slot])+class QueueProtocol(Protocol):+ """Protocol for downstream queues of ``ScrapyPriorityQueue``."""++ def push(self, request: Request) -> None: ...++ def pop(self) -> Optional[Request]: ...++ def close(self) -> None: ...++ def __len__(self) -> int: ...++class ScrapyPriorityQueue:"""A priority queue implemented using multiple internal queues (typically,FIFO queues). It uses one internal queue for each priority value. The internal@@ -51,18 +83,30 @@ class ScrapyPriorityQueue:"""@classmethod- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):+ def from_crawler(+ cls,+ crawler: Crawler,+ downstream_queue_cls: Type[QueueProtocol],+ key: str,+ startprios: Iterable[int] = (),+ ) -> Self:return cls(crawler, downstream_queue_cls, key, startprios)- def __init__(self, crawler, downstream_queue_cls, key, startprios=()):- self.crawler = crawler- self.downstream_queue_cls = downstream_queue_cls- self.key = key- self.queues = {}- self.curprio = None+ def __init__(+ self,+ crawler: Crawler,+ downstream_queue_cls: Type[QueueProtocol],+ key: str,+ startprios: Iterable[int] = (),+ ):+ self.crawler: Crawler = crawler+ self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls+ self.key: str = key+ self.queues: Dict[int, QueueProtocol] = {}+ self.curprio: Optional[int] = Noneself.init_prios(startprios)- def init_prios(self, startprios):+ def init_prios(self, startprios: Iterable[int]) -> None:if not startprios:return@@ -71,17 +115,17 @@ class ScrapyPriorityQueue:self.curprio = min(startprios)- def qfactory(self, key):+ def qfactory(self, key: int) -> QueueProtocol:return build_from_crawler(self.downstream_queue_cls,self.crawler,self.key + "/" + str(key),)- def priority(self, request):+ def priority(self, request: Request) -> int:return -request.priority- def push(self, request):+ def push(self, request: Request) -> None:priority = self.priority(request)if priority not in self.queues:self.queues[priority] = self.qfactory(priority)@@ -90,9 +134,9 @@ class ScrapyPriorityQueue:if self.curprio is None or priority < self.curprio:self.curprio = priority- def pop(self):+ def pop(self) -> Optional[Request]:if self.curprio is None:- return+ return Noneq = self.queues[self.curprio]m = q.pop()if not q:@@ -102,7 +146,7 @@ class ScrapyPriorityQueue:self.curprio = min(prios) if prios else Nonereturn m- def peek(self):+ def peek(self) -> Optional[Request]:"""Returns the next object to be returned by :meth:`pop`,but without removing it from the queue.@@ -112,30 +156,32 @@ class ScrapyPriorityQueue:if self.curprio is None:return Nonequeue = self.queues[self.curprio]- return queue.peek()+ # Protocols can't declare optional members+ return cast(Request, queue.peek()) # type: ignore[attr-defined]- def close(self):- active = []+ def close(self) -> List[int]:+ active: List[int] = []for p, q in self.queues.items():active.append(p)q.close()return active- def __len__(self):+ def __len__(self) -> int:return sum(len(x) for x in self.queues.values()) if self.queues else 0class DownloaderInterface:- def __init__(self, crawler):- self.downloader = crawler.engine.downloader+ def __init__(self, crawler: Crawler):+ assert crawler.engine+ self.downloader: Downloader = crawler.engine.downloader- def stats(self, possible_slots):+ def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:return [(self._active_downloads(slot), slot) for slot in possible_slots]- def get_slot_key(self, request):+ def get_slot_key(self, request: Request) -> str:return self.downloader._get_slot_key(request, None)- def _active_downloads(self, slot):+ def _active_downloads(self, slot: str) -> int:"""Return a number of requests in a Downloader for a given slot"""if slot not in self.downloader.slots:return 0@@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue:"""@classmethod- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):+ def from_crawler(+ cls,+ crawler: Crawler,+ downstream_queue_cls: Type[QueueProtocol],+ key: str,+ startprios: Optional[Dict[str, Iterable[int]]] = None,+ ) -> Self:return cls(crawler, downstream_queue_cls, key, startprios)- def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):+ def __init__(+ self,+ crawler: Crawler,+ downstream_queue_cls: Type[QueueProtocol],+ key: str,+ slot_startprios: Optional[Dict[str, Iterable[int]]] = None,+ ):if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'@@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue:"queue class can be resumed.")- self._downloader_interface = DownloaderInterface(crawler)- self.downstream_queue_cls = downstream_queue_cls- self.key = key- self.crawler = crawler+ self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)+ self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls+ self.key: str = key+ self.crawler: Crawler = crawler- self.pqueues = {} # slot -> priority queue+ self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queuefor slot, startprios in (slot_startprios or {}).items():self.pqueues[slot] = self.pqfactory(slot, startprios)- def pqfactory(self, slot, startprios=()):+ def pqfactory(+ self, slot: str, startprios: Iterable[int] = ()+ ) -> ScrapyPriorityQueue:return ScrapyPriorityQueue(self.crawler,self.downstream_queue_cls,@@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue:startprios,)- def pop(self):+ def pop(self) -> Optional[Request]:stats = self._downloader_interface.stats(self.pqueues)if not stats:- return+ return Noneslot = min(stats)[1]queue = self.pqueues[slot]@@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue:del self.pqueues[slot]return request- def push(self, request):+ def push(self, request: Request) -> None:slot = self._downloader_interface.get_slot_key(request)if slot not in self.pqueues:self.pqueues[slot] = self.pqfactory(slot)queue = self.pqueues[slot]queue.push(request)- def peek(self):+ def peek(self) -> Optional[Request]:"""Returns the next object to be returned by :meth:`pop`,but without removing it from the queue.@@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue:queue = self.pqueues[slot]return queue.peek()- def close(self):+ def close(self) -> Dict[str, List[int]]:active = {slot: queue.close() for slot, queue in self.pqueues.items()}self.pqueues.clear()return active- def __len__(self):+ def __len__(self) -> int:return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0- def __contains__(self, slot):+ def __contains__(self, slot: str) -> bool:return slot in self.pqueues