Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add parameters to misc generics.

Andrey Rakhmatullin751c91e614b9
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4eca03800..4cb4454e3 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst
"""
+from __future__ import annotations
+
import logging
from time import time
from typing import (
@@ -51,15 +53,15 @@ class Slot:
self,
start_requests: Iterable[Request],
close_if_idle: bool,
- nextcall: CallLaterOnce,
- scheduler: "BaseScheduler",
+ nextcall: CallLaterOnce[None],
+ scheduler: BaseScheduler,
) -> None:
self.closing: Optional[Deferred] = None
self.inprogress: Set[Request] = set()
self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
self.close_if_idle: bool = close_if_idle
- self.nextcall: CallLaterOnce = nextcall
- self.scheduler: "BaseScheduler" = scheduler
+ self.nextcall: CallLaterOnce[None] = nextcall
+ self.scheduler: BaseScheduler = scheduler
self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
def add_request(self, request: Request) -> None:
@@ -84,8 +86,8 @@ class Slot:
class ExecutionEngine:
- def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
- self.crawler: "Crawler" = crawler
+ def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None:
+ self.crawler: Crawler = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
@@ -94,7 +96,7 @@ class ExecutionEngine:
self.spider: Optional[Spider] = None
self.running: bool = False
self.paused: bool = False
- self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
+ self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class(
crawler.settings
)
downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
@@ -103,10 +105,10 @@ class ExecutionEngine:
self._spider_closed_callback: Callable = spider_closed_callback
self.start_time: Optional[float] = None
- def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
+ def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:
from scrapy.core.scheduler import BaseScheduler
- scheduler_cls: Type = load_object(settings["SCHEDULER"])
+ scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
f"The provided scheduler class ({settings['SCHEDULER']})"
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 063835b17..f2f1cb0b8 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -3,7 +3,7 @@ import itertools
import logging
from collections import deque
from ipaddress import IPv4Address, IPv6Address
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Deque, Dict, List, Optional, Union
from h2.config import H2Configuration
from h2.connection import H2Connection
@@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# If requests are received before connection is made we keep
# all requests in a pool and send them as the connection is made
- self._pending_request_stream_pool: deque = deque()
+ self._pending_request_stream_pool: Deque[Stream] = deque()
# Save an instance of errors raised which lead to losing the connection
# We pass these instances to the streams ResponseFailed() failure
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index b2118495f..d06887610 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
- def __init__(self, seq: Sequence):
- self.seq: Sequence = seq
+ def __init__(self, seq: Sequence[Any]):
+ self.seq: Sequence[Any] = seq
def __contains__(self, item: Any) -> bool:
return item not in self.seq
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 3db7acf81..fc1eb4f69 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -148,7 +148,7 @@ def to_bytes(
def re_rsearch(
- pattern: Union[str, Pattern], text: str, chunk_size: int = 1024
+ pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024
) -> Optional[Tuple[int, int]]:
"""
This function does a reverse search in a text using a regular expression
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 7a8c5c859..268d8d4be 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,7 +7,7 @@ import os
from importlib import import_module
from pathlib import Path
from posixpath import split
-from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar
from unittest import TestCase, mock
from twisted.internet.defer import Deferred
@@ -17,6 +17,8 @@ from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.utils.boto import is_botocore_available
+_T = TypeVar("_T")
+
def assert_gcs_environ() -> None:
if "GCS_PROJECT_ID" not in os.environ:
@@ -118,8 +120,8 @@ def assert_samelines(
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
-def get_from_asyncio_queue(value: Any) -> Coroutine:
- q: asyncio.Queue = asyncio.Queue()
+def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
+ q: asyncio.Queue[_T] = asyncio.Queue()
getter = q.get()
q.put_nowait(value)
return getter