Scrapy, a fast high-level web crawling & scraping framework for Python.
Add parameters to misc generics.
Andrey Rakhmatullin751c91e614b9
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 4eca03800..4cb4454e3 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst"""+from __future__ import annotations+import loggingfrom time import timefrom typing import (@@ -51,15 +53,15 @@ class Slot:self,start_requests: Iterable[Request],close_if_idle: bool,- nextcall: CallLaterOnce,- scheduler: "BaseScheduler",+ nextcall: CallLaterOnce[None],+ scheduler: BaseScheduler,) -> None:self.closing: Optional[Deferred] = Noneself.inprogress: Set[Request] = set()self.start_requests: Optional[Iterator[Request]] = iter(start_requests)self.close_if_idle: bool = close_if_idle- self.nextcall: CallLaterOnce = nextcall- self.scheduler: "BaseScheduler" = scheduler+ self.nextcall: CallLaterOnce[None] = nextcall+ self.scheduler: BaseScheduler = schedulerself.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)def add_request(self, request: Request) -> None:@@ -84,8 +86,8 @@ class Slot:class ExecutionEngine:- def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:- self.crawler: "Crawler" = crawler+ def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None:+ self.crawler: Crawler = crawlerself.settings: Settings = crawler.settingsself.signals: SignalManager = crawler.signalsassert crawler.logformatter@@ -94,7 +96,7 @@ class ExecutionEngine:self.spider: Optional[Spider] = Noneself.running: bool = Falseself.paused: bool = False- self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(+ self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class(crawler.settings)downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])@@ -103,10 +105,10 @@ class ExecutionEngine:self._spider_closed_callback: Callable = spider_closed_callbackself.start_time: Optional[float] = None- def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:+ def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:from scrapy.core.scheduler import BaseScheduler- scheduler_cls: Type = load_object(settings["SCHEDULER"])+ scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"])if not issubclass(scheduler_cls, BaseScheduler):raise TypeError(f"The provided scheduler class ({settings['SCHEDULER']})"diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 063835b17..f2f1cb0b8 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -3,7 +3,7 @@ import itertoolsimport loggingfrom collections import dequefrom ipaddress import IPv4Address, IPv6Address-from typing import Any, Dict, List, Optional, Union+from typing import Any, Deque, Dict, List, Optional, Unionfrom h2.config import H2Configurationfrom h2.connection import H2Connection@@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):# If requests are received before connection is made we keep# all requests in a pool and send them as the connection is made- self._pending_request_stream_pool: deque = deque()+ self._pending_request_stream_pool: Deque[Stream] = deque()# Save an instance of errors raised which lead to losing the connection# We pass these instances to the streams ResponseFailed() failurediff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex b2118495f..d06887610 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):class SequenceExclude:"""Object to test if an item is NOT within some sequence."""- def __init__(self, seq: Sequence):- self.seq: Sequence = seq+ def __init__(self, seq: Sequence[Any]):+ self.seq: Sequence[Any] = seqdef __contains__(self, item: Any) -> bool:return item not in self.seqdiff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 3db7acf81..fc1eb4f69 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -148,7 +148,7 @@ def to_bytes(def re_rsearch(- pattern: Union[str, Pattern], text: str, chunk_size: int = 1024+ pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024) -> Optional[Tuple[int, int]]:"""This function does a reverse search in a text using a regular expressiondiff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 7a8c5c859..268d8d4be 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -7,7 +7,7 @@ import osfrom importlib import import_modulefrom pathlib import Pathfrom posixpath import split-from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVarfrom unittest import TestCase, mockfrom twisted.internet.defer import Deferred@@ -17,6 +17,8 @@ from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.utils.boto import is_botocore_available+_T = TypeVar("_T")+def assert_gcs_environ() -> None:if "GCS_PROJECT_ID" not in os.environ:@@ -118,8 +120,8 @@ def assert_samelines(testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)-def get_from_asyncio_queue(value: Any) -> Coroutine:- q: asyncio.Queue = asyncio.Queue()+def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:+ q: asyncio.Queue[_T] = asyncio.Queue()getter = q.get()q.put_nowait(value)return getter