Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Full typing for scrapy/pqueues.py.

Andrey Rakhmatullin0c8e21b8acfc
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index ecd3e8b56..f88da41ea 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -143,7 +143,7 @@ class Downloader:
return key, self.slots[key]
- def _get_slot_key(self, request: Request, spider: Spider) -> str:
+ def _get_slot_key(self, request: Request, spider: Any) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return cast(str, request.meta[self.DOWNLOAD_SLOT])
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 593667f1f..213ad590d 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,12 +1,32 @@
+from __future__ import annotations
+
import hashlib
import logging
-
+from typing import (
+ TYPE_CHECKING,
+ Dict,
+ Iterable,
+ List,
+ Optional,
+ Protocol,
+ Tuple,
+ Type,
+ cast,
+)
+
+from scrapy import Request
+from scrapy.core.downloader import Downloader
+from scrapy.crawler import Crawler
from scrapy.utils.misc import build_from_crawler
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
logger = logging.getLogger(__name__)
-def _path_safe(text):
+def _path_safe(text: str) -> str:
"""
Return a filesystem-safe version of a string ``text``
@@ -24,6 +44,18 @@ def _path_safe(text):
return "-".join([pathable_slot, unique_slot])
+class QueueProtocol(Protocol):
+ """Protocol for downstream queues of ``ScrapyPriorityQueue``."""
+
+ def push(self, request: Request) -> None: ...
+
+ def pop(self) -> Optional[Request]: ...
+
+ def close(self) -> None: ...
+
+ def __len__(self) -> int: ...
+
+
class ScrapyPriorityQueue:
"""A priority queue implemented using multiple internal queues (typically,
FIFO queues). It uses one internal queue for each priority value. The internal
@@ -51,18 +83,30 @@ class ScrapyPriorityQueue:
"""
@classmethod
- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+ def from_crawler(
+ cls,
+ crawler: Crawler,
+ downstream_queue_cls: Type[QueueProtocol],
+ key: str,
+ startprios: Iterable[int] = (),
+ ) -> Self:
return cls(crawler, downstream_queue_cls, key, startprios)
- def __init__(self, crawler, downstream_queue_cls, key, startprios=()):
- self.crawler = crawler
- self.downstream_queue_cls = downstream_queue_cls
- self.key = key
- self.queues = {}
- self.curprio = None
+ def __init__(
+ self,
+ crawler: Crawler,
+ downstream_queue_cls: Type[QueueProtocol],
+ key: str,
+ startprios: Iterable[int] = (),
+ ):
+ self.crawler: Crawler = crawler
+ self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
+ self.key: str = key
+ self.queues: Dict[int, QueueProtocol] = {}
+ self.curprio: Optional[int] = None
self.init_prios(startprios)
- def init_prios(self, startprios):
+ def init_prios(self, startprios: Iterable[int]) -> None:
if not startprios:
return
@@ -71,17 +115,17 @@ class ScrapyPriorityQueue:
self.curprio = min(startprios)
- def qfactory(self, key):
+ def qfactory(self, key: int) -> QueueProtocol:
return build_from_crawler(
self.downstream_queue_cls,
self.crawler,
self.key + "/" + str(key),
)
- def priority(self, request):
+ def priority(self, request: Request) -> int:
return -request.priority
- def push(self, request):
+ def push(self, request: Request) -> None:
priority = self.priority(request)
if priority not in self.queues:
self.queues[priority] = self.qfactory(priority)
@@ -90,9 +134,9 @@ class ScrapyPriorityQueue:
if self.curprio is None or priority < self.curprio:
self.curprio = priority
- def pop(self):
+ def pop(self) -> Optional[Request]:
if self.curprio is None:
- return
+ return None
q = self.queues[self.curprio]
m = q.pop()
if not q:
@@ -102,7 +146,7 @@ class ScrapyPriorityQueue:
self.curprio = min(prios) if prios else None
return m
- def peek(self):
+ def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@@ -112,30 +156,32 @@ class ScrapyPriorityQueue:
if self.curprio is None:
return None
queue = self.queues[self.curprio]
- return queue.peek()
+ # Protocols can't declare optional members
+ return cast(Request, queue.peek()) # type: ignore[attr-defined]
- def close(self):
- active = []
+ def close(self) -> List[int]:
+ active: List[int] = []
for p, q in self.queues.items():
active.append(p)
q.close()
return active
- def __len__(self):
+ def __len__(self) -> int:
return sum(len(x) for x in self.queues.values()) if self.queues else 0
class DownloaderInterface:
- def __init__(self, crawler):
- self.downloader = crawler.engine.downloader
+ def __init__(self, crawler: Crawler):
+ assert crawler.engine
+ self.downloader: Downloader = crawler.engine.downloader
- def stats(self, possible_slots):
+ def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:
return [(self._active_downloads(slot), slot) for slot in possible_slots]
- def get_slot_key(self, request):
+ def get_slot_key(self, request: Request) -> str:
return self.downloader._get_slot_key(request, None)
- def _active_downloads(self, slot):
+ def _active_downloads(self, slot: str) -> int:
"""Return a number of requests in a Downloader for a given slot"""
if slot not in self.downloader.slots:
return 0
@@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue:
"""
@classmethod
- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+ def from_crawler(
+ cls,
+ crawler: Crawler,
+ downstream_queue_cls: Type[QueueProtocol],
+ key: str,
+ startprios: Optional[Dict[str, Iterable[int]]] = None,
+ ) -> Self:
return cls(crawler, downstream_queue_cls, key, startprios)
- def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
+ def __init__(
+ self,
+ crawler: Crawler,
+ downstream_queue_cls: Type[QueueProtocol],
+ key: str,
+ slot_startprios: Optional[Dict[str, Iterable[int]]] = None,
+ ):
if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
raise ValueError(
f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
@@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue:
"queue class can be resumed."
)
- self._downloader_interface = DownloaderInterface(crawler)
- self.downstream_queue_cls = downstream_queue_cls
- self.key = key
- self.crawler = crawler
+ self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)
+ self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
+ self.key: str = key
+ self.crawler: Crawler = crawler
- self.pqueues = {} # slot -> priority queue
+ self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue
for slot, startprios in (slot_startprios or {}).items():
self.pqueues[slot] = self.pqfactory(slot, startprios)
- def pqfactory(self, slot, startprios=()):
+ def pqfactory(
+ self, slot: str, startprios: Iterable[int] = ()
+ ) -> ScrapyPriorityQueue:
return ScrapyPriorityQueue(
self.crawler,
self.downstream_queue_cls,
@@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue:
startprios,
)
- def pop(self):
+ def pop(self) -> Optional[Request]:
stats = self._downloader_interface.stats(self.pqueues)
if not stats:
- return
+ return None
slot = min(stats)[1]
queue = self.pqueues[slot]
@@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue:
del self.pqueues[slot]
return request
- def push(self, request):
+ def push(self, request: Request) -> None:
slot = self._downloader_interface.get_slot_key(request)
if slot not in self.pqueues:
self.pqueues[slot] = self.pqfactory(slot)
queue = self.pqueues[slot]
queue.push(request)
- def peek(self):
+ def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue:
queue = self.pqueues[slot]
return queue.peek()
- def close(self):
+ def close(self) -> Dict[str, List[int]]:
active = {slot: queue.close() for slot, queue in self.pqueues.items()}
self.pqueues.clear()
return active
- def __len__(self):
+ def __len__(self) -> int:
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
- def __contains__(self, slot):
+ def __contains__(self, slot: str) -> bool:
return slot in self.pqueues