Scrapy, a fast high-level web crawling & scraping framework for Python.
Add parameters to most Deferred in scrapy/core. (#6395)
Andrey Rakhmatullin1282ddf8f772
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 0ab3bdb77..41f729ed9 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -1,9 +1,22 @@+from __future__ import annotations+import randomimport warningsfrom collections import dequefrom datetime import datetimefrom time import time-from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast+from typing import (+ TYPE_CHECKING,+ Any,+ Deque,+ Dict,+ Optional,+ Set,+ Tuple,+ TypeVar,+ Union,+ cast,+)from twisted.internet import taskfrom twisted.internet.defer import Deferred@@ -22,6 +35,8 @@ from scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from scrapy.crawler import Crawler+_T = TypeVar("_T")+class Slot:"""Downloader slot"""@@ -40,7 +55,7 @@ class Slot:self.throttle = throttleself.active: Set[Request] = set()- self.queue: Deque[Tuple[Request, Deferred]] = deque()+ self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque()self.transferring: Set[Request] = set()self.lastseen: float = 0self.latercall = None@@ -93,7 +108,7 @@ def _get_concurrency_delay(class Downloader:DOWNLOAD_SLOT = "download_slot"- def __init__(self, crawler: "Crawler"):+ def __init__(self, crawler: Crawler):self.settings: BaseSettings = crawler.settingsself.signals: SignalManager = crawler.signalsself.slots: Dict[str, Slot] = {}@@ -114,13 +129,17 @@ class Downloader:"DOWNLOAD_SLOTS", {})- def fetch(self, request: Request, spider: Spider) -> Deferred:- def _deactivate(response: Response) -> Response:+ def fetch(+ self, request: Request, spider: Spider+ ) -> Deferred[Union[Response, Request]]:+ def _deactivate(response: _T) -> _T:self.active.remove(request)return responseself.active.add(request)- dfd = self.middleware.download(self._enqueue_request, request, spider)+ dfd: Deferred[Union[Response, Request]] = self.middleware.download(+ self._enqueue_request, request, spider+ )return dfd.addBoth(_deactivate)def needs_backout(self) -> bool:@@ -163,7 +182,7 @@ class Downloader:)return self.get_slot_key(request)- def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:+ def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]:key, slot = self._get_slot(request, spider)request.meta[self.DOWNLOAD_SLOT] = key@@ -175,7 +194,7 @@ class Downloader:self.signals.send_catch_log(signal=signals.request_reached_downloader, request=request, spider=spider)- deferred: Deferred = Deferred().addBoth(_deactivate)+ deferred: Deferred[Response] = Deferred().addBoth(_deactivate)slot.queue.append((request, deferred))self._process_queue(spider, slot)return deferred@@ -208,11 +227,15 @@ class Downloader:self._process_queue(spider, slot)break- def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred:+ def _download(+ self, slot: Slot, request: Request, spider: Spider+ ) -> Deferred[Response]:# The order is very important for the following deferreds. Do not change!# 1. Create the download deferred- dfd = mustbe_deferred(self.handlers.download_request, request, spider)+ dfd: Deferred[Response] = mustbe_deferred(+ self.handlers.download_request, request, spider+ )# 2. Notify response_downloaded listeners about the recent download# before querying queue for next request@@ -233,7 +256,7 @@ class Downloader:# middleware itself)slot.transferring.add(request)- def finish_transferring(_: Any) -> Any:+ def finish_transferring(_: _T) -> _T:slot.transferring.remove(request)self._process_queue(spider, slot)self.signals.send_catch_log(diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 5ec5ef6db..ebc4898b5 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -3,13 +3,25 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast+from typing import (+ TYPE_CHECKING,+ Any,+ Callable,+ Dict,+ Generator,+ Optional,+ Protocol,+ Type,+ Union,+ cast,+)from twisted.internet import deferfrom twisted.internet.defer import Deferredfrom scrapy import Request, Spider, signalsfrom scrapy.exceptions import NotConfigured, NotSupported+from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_values@@ -20,13 +32,21 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+class DownloadHandlerProtocol(Protocol):+ def download_request(+ self, request: Request, spider: Spider+ ) -> Deferred[Response]: ...++class DownloadHandlers:def __init__(self, crawler: Crawler):self._crawler: Crawler = crawlerself._schemes: Dict[str, Union[str, Callable[..., Any]]] = ({}) # stores acceptable schemes on instancing- self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes+ self._handlers: Dict[str, DownloadHandlerProtocol] = (+ {}+ ) # stores instanced handlers for schemesself._notconfigured: Dict[str, str] = {} # remembers failed handlershandlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values(cast(@@ -40,7 +60,7 @@ class DownloadHandlers:crawler.signals.connect(self._close, signals.engine_stopped)- def _get_handler(self, scheme: str) -> Any:+ def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]:"""Lazy-load the downloadhandler for a schemeonly on the first request for that scheme."""@@ -54,10 +74,12 @@ class DownloadHandlers:return self._load_handler(scheme)- def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any:+ def _load_handler(+ self, scheme: str, skip_lazy: bool = False+ ) -> Optional[DownloadHandlerProtocol]:path = self._schemes[scheme]try:- dhcls = load_object(path)+ dhcls: Type[DownloadHandlerProtocol] = load_object(path)if skip_lazy and getattr(dhcls, "lazy", True):return Nonedh = build_from_crawler(@@ -80,17 +102,17 @@ class DownloadHandlers:self._handlers[scheme] = dhreturn dh- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:scheme = urlparse_cached(request).schemehandler = self._get_handler(scheme)if not handler:raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")- return cast(Deferred, handler.download_request(request, spider))+ return handler.download_request(request, spider)@defer.inlineCallbacks- def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]:+ def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]:for dh in self._handlers.values():if hasattr(dh, "close"):yield dh.close()diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 77dcf3c38..724717ffd 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -91,7 +91,7 @@ class FTPDownloadHandler:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:from twisted.internet import reactorparsed_url = urlparse_cached(request)@@ -103,10 +103,14 @@ class FTPDownloadHandler:creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)- dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)+ dfd: Deferred[FTPClient] = creator.connectTCP(+ parsed_url.hostname, parsed_url.port or 21+ )return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))- def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:+ def gotClient(+ self, client: FTPClient, request: Request, filepath: str+ ) -> Deferred[Response]:self.client = clientprotocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))d = client.retrieveFile(filepath, protocol)diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.pyindex da9559525..3c4e48abb 100644--- a/scrapy/core/downloader/handlers/http10.py+++ b/scrapy/core/downloader/handlers/http10.py@@ -9,6 +9,7 @@ from twisted.internet.defer import Deferredfrom scrapy import Request, Spiderfrom scrapy.crawler import Crawler+from scrapy.http import Responsefrom scrapy.settings import BaseSettingsfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import to_unicode@@ -38,7 +39,7 @@ class HTTP10DownloadHandler:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings, crawler)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:"""Return a deferred for the HTTP download"""factory = self.HTTPClientFactory(request)self._connect(factory)diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 5e84be6ba..e2ad8f59a 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -8,7 +8,7 @@ import refrom contextlib import suppressfrom io import BytesIOfrom time import time-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Unionfrom urllib.parse import urldefrag, urlunparsefrom twisted.internet import ssl@@ -38,12 +38,22 @@ from scrapy.settings import BaseSettingsfrom scrapy.utils.python import to_bytes, to_unicodeif TYPE_CHECKING:- # typing.Self requires Python 3.11- from typing_extensions import Self-+ # typing.NotRequired and typing.Self require Python 3.11+ from typing_extensions import NotRequired, Selflogger = logging.getLogger(__name__)+_T = TypeVar("_T")+++class _ResultT(TypedDict):+ txresponse: TxResponse+ body: bytes+ flags: Optional[List[str]]+ certificate: Optional[ssl.Certificate]+ ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None]+ failure: NotRequired[Optional[Failure]]+class HTTP11DownloadHandler:lazy = False@@ -71,7 +81,7 @@ class HTTP11DownloadHandler:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings, crawler)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:"""Return a deferred for the HTTP download"""agent = ScrapyAgent(contextFactory=self._contextFactory,@@ -83,10 +93,10 @@ class HTTP11DownloadHandler:)return agent.download_request(request)- def close(self) -> Deferred:+ def close(self) -> Deferred[None]:from twisted.internet import reactor- d: Deferred = self._pool.closeCachedConnections()+ d: Deferred[None] = self._pool.closeCachedConnections()# closeCachedConnections will hang on network or server issues, so# we'll manually timeout the deferred.#@@ -97,7 +107,7 @@ class HTTP11DownloadHandler:# issue a callback after `_disconnect_timeout` seconds.delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])- def cancel_delayed_call(result: Any) -> Any:+ def cancel_delayed_call(result: _T) -> _T:if delayed_call.active():delayed_call.cancel()return result@@ -137,7 +147,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):):proxyHost, proxyPort, self._proxyAuthHeader = proxyConfsuper().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)- self._tunnelReadyDeferred: Deferred = Deferred()+ self._tunnelReadyDeferred: Deferred[Protocol] = Deferred()self._tunneledHost: str = hostself._tunneledPort: int = portself._contextFactory: IPolicyForHTTPS = contextFactory@@ -198,7 +208,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):"""Propagates the errback to the appropriate deferred."""self._tunnelReadyDeferred.errback(reason)- def connect(self, protocolFactory: Factory) -> Deferred:+ def connect(self, protocolFactory: Factory) -> Deferred[Protocol]:self._protocolFactory = protocolFactoryconnectDeferred = super().connect(protocolFactory)connectDeferred.addCallback(self.requestTunnel)@@ -271,7 +281,7 @@ class TunnelingAgent(Agent):headers: Optional[TxHeaders],bodyProducer: Optional[IBodyProducer],requestPath: bytes,- ) -> Deferred:+ ) -> Deferred[TxResponse]:# proxy host and port are required for HTTP pool `key`# otherwise, same remote host connection request could reuse# a cached tunneled connection to a different proxy@@ -310,7 +320,7 @@ class ScrapyProxyAgent(Agent):uri: bytes,headers: Optional[TxHeaders] = None,bodyProducer: Optional[IBodyProducer] = None,- ) -> Deferred:+ ) -> Deferred[TxResponse]:"""Issue a new request via the configured proxy."""@@ -394,7 +404,7 @@ class ScrapyAgent:pool=self._pool,)- def download_request(self, request: Request) -> Deferred:+ def download_request(self, request: Request) -> Deferred[Response]:from twisted.internet import reactortimeout = request.meta.get("download_timeout") or self._connectTimeout@@ -411,22 +421,20 @@ class ScrapyAgent:else:bodyproducer = Nonestart_time = time()- d: Deferred = agent.request(+ d: Deferred[TxResponse] = agent.request(method, to_bytes(url, encoding="ascii"), headers, bodyproducer)# set download latencyd.addCallback(self._cb_latency, request, start_time)# response body is ready to be consumed- d.addCallback(self._cb_bodyready, request)- d.addCallback(self._cb_bodydone, request, url)+ d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)+ d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url)# check download timeout- self._timeout_cl = reactor.callLater(timeout, d.cancel)- d.addBoth(self._cb_timeout, request, url, timeout)- return d+ self._timeout_cl = reactor.callLater(timeout, d3.cancel)+ d3.addBoth(self._cb_timeout, request, url, timeout)+ return d3- def _cb_timeout(- self, result: Any, request: Request, url: str, timeout: float- ) -> Any:+ def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:if self._timeout_cl.active():self._timeout_cl.cancel()return result@@ -437,7 +445,7 @@ class ScrapyAgent:raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")- def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:+ def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:request.meta["download_latency"] = time() - start_timereturn result@@ -451,7 +459,7 @@ class ScrapyAgent:def _cb_bodyready(self, txresponse: TxResponse, request: Request- ) -> Union[Dict[str, Any], Deferred]:+ ) -> Union[_ResultT, Deferred[_ResultT]]:headers_received_result = self._crawler.signals.send_catch_log(signal=signals.headers_received,headers=self._headers_from_twisted_response(txresponse),@@ -520,7 +528,7 @@ class ScrapyAgent:# Abort connection immediately.txresponse._transport._producer.abortConnection()- d: Deferred = Deferred(_cancel)+ d: Deferred[_ResultT] = Deferred(_cancel)txresponse.deliverBody(_ResponseReader(finished=d,@@ -539,7 +547,7 @@ class ScrapyAgent:return ddef _cb_bodydone(- self, result: Dict[str, Any], request: Request, url: str+ self, result: _ResultT, request: Request, url: str) -> Union[Response, Failure]:headers = self._headers_from_twisted_response(result["txresponse"])respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])@@ -559,8 +567,9 @@ class ScrapyAgent:protocol=protocol,)if result.get("failure"):+ assert result["failure"]result["failure"].value.response = response- return cast(Failure, result["failure"])+ return result["failure"]return response@@ -570,7 +579,7 @@ class _RequestBodyProducer:self.body = bodyself.length = len(body)- def startProducing(self, consumer: IConsumer) -> Deferred:+ def startProducing(self, consumer: IConsumer) -> Deferred[None]:consumer.write(self.body)return succeed(None)@@ -584,7 +593,7 @@ class _RequestBodyProducer:class _ResponseReader(Protocol):def __init__(self,- finished: Deferred,+ finished: Deferred[_ResultT],txresponse: TxResponse,request: Request,maxsize: int,@@ -592,7 +601,7 @@ class _ResponseReader(Protocol):fail_on_dataloss: bool,crawler: Crawler,):- self._finished: Deferred = finished+ self._finished: Deferred[_ResultT] = finishedself._txresponse: TxResponse = txresponseself._request: Request = requestself._bodybuf: BytesIO = BytesIO()diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 16fc1e3ae..2ac4eca86 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -37,7 +37,7 @@ class H2DownloadHandler:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings, crawler)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:agent = ScrapyH2Agent(context_factory=self._context_factory,pool=self._pool,@@ -98,7 +98,7 @@ class ScrapyH2Agent:pool=self._pool,)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:from twisted.internet import reactortimeout = request.meta.get("download_timeout") or self._connect_timeoutdiff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 1a3d36f45..0ad340721 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -8,6 +8,7 @@ from scrapy import Request, Spiderfrom scrapy.core.downloader.handlers.http import HTTPDownloadHandlerfrom scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.http import Responsefrom scrapy.settings import BaseSettingsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.httpobj import urlparse_cached@@ -76,7 +77,7 @@ class S3DownloadHandler:def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:return cls(crawler.settings, crawler=crawler, **kwargs)- def download_request(self, request: Request, spider: Spider) -> Deferred:+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:p = urlparse_cached(request)scheme = "https" if request.meta.get("is_secure") else "http"bucket = p.hostnamediff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 52ebe4e22..2d8af114f 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -4,6 +4,8 @@ Downloader Middleware managerSee documentation in docs/topics/downloader-middleware.rst"""+from __future__ import annotations+from typing import Any, Callable, Generator, List, Union, castfrom twisted.internet.defer import Deferred, inlineCallbacks@@ -34,10 +36,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):self.methods["process_exception"].appendleft(mw.process_exception)def download(- self, download_func: Callable, request: Request, spider: Spider- ) -> Deferred:+ self,+ download_func: Callable[[Request, Spider], Deferred[Response]],+ request: Request,+ spider: Spider,+ ) -> Deferred[Union[Response, Request]]:@inlineCallbacks- def process_request(request: Request) -> Generator[Deferred, Any, Any]:+ def process_request(+ request: Request,+ ) -> Generator[Deferred[Any], Any, Union[Response, Request]]:for method in self.methods["process_request"]:method = cast(Callable, method)response = yield deferred_from_coro(@@ -52,12 +59,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):)if response:return response- return (yield download_func(request=request, spider=spider))+ return (yield download_func(request, spider))@inlineCallbacksdef process_response(response: Union[Response, Request]- ) -> Generator[Deferred, Any, Union[Response, Request]]:+ ) -> Generator[Deferred[Any], Any, Union[Response, Request]]:if response is None:raise TypeError("Received None in process_response")elif isinstance(response, Request):@@ -80,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):@inlineCallbacksdef process_exception(failure: Failure,- ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]:+ ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]:exception = failure.valuefor method in self.methods["process_exception"]:method = cast(Callable, method)@@ -98,7 +105,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):return responsereturn failure- deferred = mustbe_deferred(process_request, request)+ deferred: Deferred[Union[Response, Request]] = mustbe_deferred(+ process_request, request+ )deferred.addErrback(process_exception)deferred.addCallback(process_response)return deferreddiff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex bb1f73805..08a1d7c71 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -8,7 +8,7 @@ from twisted.internet.protocol import ClientFactoryfrom twisted.web.http import HTTPClientfrom scrapy import Request-from scrapy.http import Headers+from scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode@@ -145,7 +145,7 @@ class ScrapyHTTPClientFactory(ClientFactory):self.response_headers: Optional[Headers] = Noneself.timeout: float = request.meta.get("download_timeout") or timeoutself.start_time: float = time()- self.deferred: defer.Deferred = defer.Deferred().addCallback(+ self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(self._build_response, request)@@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):# needed to add the callback _waitForDisconnect.# Specifically this avoids the AttributeError exception when# clientConnectionFailed method is called.- self._disconnectedDeferred: defer.Deferred = defer.Deferred()+ self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred()self._set_connection_attributes(request)diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex dededf99d..4ffec78b9 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -19,6 +19,7 @@ from typing import (Optional,Set,Type,+ TypeVar,Union,cast,)@@ -43,10 +44,13 @@ from scrapy.utils.reactor import CallLaterOnceif TYPE_CHECKING:from scrapy.core.scheduler import BaseScheduler+ from scrapy.core.scraper import _HandleOutputDeferredfrom scrapy.crawler import Crawlerlogger = logging.getLogger(__name__)+_T = TypeVar("_T")+class Slot:def __init__(@@ -56,7 +60,7 @@ class Slot:nextcall: CallLaterOnce[None],scheduler: BaseScheduler,) -> None:- self.closing: Optional[Deferred] = None+ self.closing: Optional[Deferred[None]] = Noneself.inprogress: Set[Request] = set()self.start_requests: Optional[Iterator[Request]] = iter(start_requests)self.close_if_idle: bool = close_if_idle@@ -71,7 +75,7 @@ class Slot:self.inprogress.remove(request)self._maybe_fire_closing()- def close(self) -> Deferred:+ def close(self) -> Deferred[None]:self.closing = Deferred()self._maybe_fire_closing()return self.closing@@ -123,20 +127,20 @@ class ExecutionEngine:return scheduler_cls@inlineCallbacks- def start(self) -> Generator[Deferred, Any, None]:+ def start(self) -> Generator[Deferred[Any], Any, None]:if self.running:raise RuntimeError("Engine already running")self.start_time = time()yield self.signals.send_catch_log_deferred(signal=signals.engine_started)self.running = True- self._closewait: Deferred = Deferred()+ self._closewait: Deferred[None] = Deferred()yield self._closewait- def stop(self) -> Deferred:+ def stop(self) -> Deferred[None]:"""Gracefully stop the execution engine"""@inlineCallbacks- def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:+ def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]:yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)self._closewait.callback(None)@@ -151,7 +155,7 @@ class ExecutionEngine:)return dfd.addBoth(_finish_stopping_engine)- def close(self) -> Deferred:+ def close(self) -> Deferred[None]:"""Gracefully close the execution engine.If it has already been started, stop it. In all cases, close the spider and the downloader.@@ -214,7 +218,7 @@ class ExecutionEngine:or self.scraper.slot.needs_backout())- def _next_request_from_scheduler(self) -> Optional[Deferred]:+ def _next_request_from_scheduler(self) -> Optional[Deferred[None]]:assert self.slot is not None # typingassert self.spider is not None # typing@@ -222,7 +226,7 @@ class ExecutionEngine:if request is None:return None- d = self._download(request)+ d: Deferred[Union[Response, Request]] = self._download(request)d.addBoth(self._handle_downloader_output, request)d.addErrback(lambda f: logger.info(@@ -236,8 +240,8 @@ class ExecutionEngine:assert self.slotself.slot.remove_request(request)- d.addBoth(_remove_request)- d.addErrback(+ d2: Deferred[None] = d.addBoth(_remove_request)+ d2.addErrback(lambda f: logger.info("Error while removing request from slot",exc_info=failure_to_exc_info(f),@@ -245,19 +249,19 @@ class ExecutionEngine:))slot = self.slot- d.addBoth(lambda _: slot.nextcall.schedule())- d.addErrback(+ d2.addBoth(lambda _: slot.nextcall.schedule())+ d2.addErrback(lambda f: logger.info("Error while scheduling new request",exc_info=failure_to_exc_info(f),extra={"spider": self.spider},))- return d+ return d2def _handle_downloader_output(self, result: Union[Request, Response, Failure], request: Request- ) -> Optional[Deferred]:+ ) -> Optional[_HandleOutputDeferred]:assert self.spider is not None # typingif not isinstance(result, (Request, Response, Failure)):@@ -319,20 +323,23 @@ class ExecutionEngine:signals.request_dropped, request=request, spider=spider)- def download(self, request: Request) -> Deferred:+ def download(self, request: Request) -> Deferred[Response]:"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""if self.spider is None:raise RuntimeError(f"No open spider to crawl: {request}")- return self._download(request).addBoth(self._downloaded, request)+ d: Deferred[Union[Response, Request]] = self._download(request)+ # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type+ d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type]+ return d2def _downloaded(self, result: Union[Response, Request, Failure], request: Request- ) -> Union[Deferred, Response, Failure]:+ ) -> Union[Deferred[Response], Response, Failure]:assert self.slot is not None # typingself.slot.remove_request(request)return self.download(result) if isinstance(result, Request) else result- def _download(self, request: Request) -> Deferred:+ def _download(self, request: Request) -> Deferred[Union[Response, Request]]:assert self.slot is not None # typingself.slot.add_request(request)@@ -359,13 +366,15 @@ class ExecutionEngine:)return result- def _on_complete(_: Any) -> Any:+ def _on_complete(_: _T) -> _T:assert self.slot is not Noneself.slot.nextcall.schedule()return _assert self.spider is not None- dwld = self.downloader.fetch(request, self.spider)+ dwld: Deferred[Union[Response, Request]] = self.downloader.fetch(+ request, self.spider+ )dwld.addCallback(_on_success)dwld.addBoth(_on_complete)return dwld@@ -376,7 +385,7 @@ class ExecutionEngine:spider: Spider,start_requests: Iterable[Request] = (),close_if_idle: bool = True,- ) -> Generator[Deferred, Any, None]:+ ) -> Generator[Deferred[Any], Any, None]:if self.slot is not None:raise RuntimeError(f"No free spider slot when opening {spider.name!r}")logger.info("Spider opened", extra={"spider": spider})@@ -422,7 +431,7 @@ class ExecutionEngine:assert isinstance(ex, CloseSpider) # typingself.close_spider(self.spider, reason=ex.reason)- def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:+ def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]:"""Close (cancel) spider and clear all its outstanding requests"""if self.slot is None:raise RuntimeError("Engine slot not assigned")diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex e3b95e977..1e586c53a 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -71,7 +71,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):"""return cls()- def open(self, spider: Spider) -> Optional[Deferred]:+ def open(self, spider: Spider) -> Optional[Deferred[None]]:"""Called when the spider is opened by the engine. It receives the spiderinstance as argument and it's useful to execute initialization code.@@ -81,7 +81,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):"""pass- def close(self, reason: str) -> Optional[Deferred]:+ def close(self, reason: str) -> Optional[Deferred[None]]:"""Called when the spider is closed by the engine. It receives the reason why the crawlfinished as argument and it's useful to execute cleaning code.@@ -216,7 +216,7 @@ class Scheduler(BaseScheduler):def has_pending_requests(self) -> bool:return len(self) > 0- def open(self, spider: Spider) -> Optional[Deferred]:+ def open(self, spider: Spider) -> Optional[Deferred[None]]:"""(1) initialize the memory queue(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory@@ -227,7 +227,7 @@ class Scheduler(BaseScheduler):self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else Nonereturn self.df.open()- def close(self, reason: str) -> Optional[Deferred]:+ def close(self, reason: str) -> Optional[Deferred[None]]:"""(1) dump pending requests to disk if there is a disk queue(2) return the result of the dupefilter's ``close`` methoddiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 3b7492838..8a9e8f687 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -12,6 +12,7 @@ from typing import (Deque,Generator,Iterable,+ Iterator,Optional,Set,Tuple,@@ -33,6 +34,7 @@ from scrapy.logformatter import LogFormatterfrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.signalmanager import SignalManagerfrom scrapy.utils.defer import (+ DeferredListResultListT,aiter_errback,defer_fail,defer_succeed,@@ -48,11 +50,16 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler-_T = TypeVar("_T")-QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]+logger = logging.getLogger(__name__)-logger = logging.getLogger(__name__)+_T = TypeVar("_T")+_ParallelResult = DeferredListResultListT[Iterator[Any]]++if TYPE_CHECKING:+ # parameterized Deferreds require Twisted 21.7.0+ _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]]+ QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred]class Slot:@@ -66,12 +73,12 @@ class Slot:self.active: Set[Request] = set()self.active_size: int = 0self.itemproc_size: int = 0- self.closing: Optional[Deferred] = None+ self.closing: Optional[Deferred[Spider]] = Nonedef add_response_request(self, result: Union[Response, Failure], request: Request- ) -> Deferred:- deferred: Deferred = Deferred()+ ) -> _HandleOutputDeferred:+ deferred: _HandleOutputDeferred = Deferred()self.queue.append((result, request, deferred))if isinstance(result, Response):self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)@@ -117,12 +124,12 @@ class Scraper:self.logformatter: LogFormatter = crawler.logformatter@inlineCallbacks- def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:+ def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]:"""Open the given spider for scraping and allocate resources for it"""self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))yield self.itemproc.open_spider(spider)- def close_spider(self, spider: Spider) -> Deferred:+ def close_spider(self, spider: Spider) -> Deferred[Spider]:"""Close a spider being scraped and release its resources"""if self.slot is None:raise RuntimeError("Scraper slot not assigned")@@ -142,12 +149,12 @@ class Scraper:def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider- ) -> Deferred:+ ) -> _HandleOutputDeferred:if self.slot is None:raise RuntimeError("Scraper slot not assigned")dfd = self.slot.add_response_request(result, request)- def finish_scraping(_: Any) -> Any:+ def finish_scraping(_: _T) -> _T:assert self.slot is not Noneself.slot.finish_response(result, request)self._check_if_closing(spider)@@ -174,7 +181,7 @@ class Scraper:def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider- ) -> Deferred:+ ) -> _HandleOutputDeferred:"""Handle the downloaded response or failure through the spider callback/errback"""@@ -182,32 +189,35 @@ class Scraper:raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")- dfd = self._scrape2(+ dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2(result, request, spider) # returns spider's processed outputdfd.addErrback(self.handle_spider_error, request, result, spider)- dfd.addCallback(+ dfd2: _HandleOutputDeferred = dfd.addCallback(self.handle_spider_output, request, cast(Response, result), spider)- return dfd+ return dfd2def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider- ) -> Deferred:+ ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:"""Handle the different cases of request's result been a Response or a Failure"""if isinstance(result, Response):- return self.spidermw.scrape_response(+ # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable+ return self.spidermw.scrape_response( # type: ignore[return-value]self.call_spider, result, request, spider)# else result is a Failuredfd = self.call_spider(result, request, spider)- return dfd.addErrback(self._log_download_errors, result, request, spider)+ dfd.addErrback(self._log_download_errors, result, request, spider)+ return dfddef call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider- ) -> Deferred:+ ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:+ dfd: Deferred[Any]if isinstance(result, Response):if getattr(result, "request", None) is None:result.request = request@@ -225,7 +235,10 @@ class Scraper:if request.errback:warn_on_generator_with_return_value(spider, request.errback)dfd.addErrback(request.errback)- return dfd.addCallback(iterate_spider_output)+ dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback(+ iterate_spider_output+ )+ return dfd2def handle_spider_error(self,@@ -262,10 +275,11 @@ class Scraper:request: Request,response: Response,spider: Spider,- ) -> Deferred:+ ) -> _HandleOutputDeferred:if not result:return defer_succeed(None)it: Union[Iterable[_T], AsyncIterable[_T]]+ dfd: Deferred[_ParallelResult]if isinstance(result, AsyncIterable):it = aiter_errback(result, self.handle_spider_error, request, response, spider@@ -290,11 +304,12 @@ class Scraper:response,spider,)- return dfd+ # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]]+ return dfd # type: ignore[return-value]def _process_spidermw_output(self, output: Any, request: Request, response: Response, spider: Spider- ) -> Optional[Deferred]:+ ) -> Optional[Deferred[Any]]:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider"""diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 58873f0d9..e792f8ca7 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -45,7 +45,9 @@ logger = logging.getLogger(__name__)_T = TypeVar("_T")-ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]+ScrapeFunc = Callable[+ [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]]+]def _isiterable(o: Any) -> bool:@@ -80,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager):response: Response,request: Request,spider: Spider,- ) -> Any:+ ) -> Union[Iterable[_T], AsyncIterable[_T]]:for method in self.methods["process_spider_input"]:method = cast(Callable, method)try:@@ -311,7 +313,7 @@ class SpiderMiddlewareManager(MiddlewareManager):response: Response,request: Request,spider: Spider,- ) -> Deferred:+ ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]:async def process_callback_output(result: Union[Iterable[_T], AsyncIterable[_T]]) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:@@ -322,12 +324,14 @@ class SpiderMiddlewareManager(MiddlewareManager):) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:return self._process_spider_exception(response, spider, _failure)- dfd: Deferred = mustbe_deferred(+ dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)- dfd.addCallback(deferred_f_from_coro_f(process_callback_output))- dfd.addErrback(process_spider_exception)- return dfd+ dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (+ dfd.addCallback(deferred_f_from_coro_f(process_callback_output))+ )+ dfd2.addErrback(process_spider_exception)+ return dfd2def process_start_requests(self, start_requests: Iterable[Request], spider: Spiderdiff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 0cfbc156f..21d649e3c 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -4,6 +4,8 @@ Item pipelineSee documentation in docs/item-pipeline.rst"""+from __future__ import annotations+from typing import Any, Listfrom twisted.internet.defer import Deferred@@ -29,5 +31,5 @@ class ItemPipelineManager(MiddlewareManager):deferred_f_from_coro_f(pipe.process_item))- def process_item(self, item: Any, spider: Spider) -> Deferred:+ def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:return self._process_chain("process_item", item, spider)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex f60b7dde8..ddb68c86b 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -46,6 +46,12 @@ if TYPE_CHECKING:_P = ParamSpec("_P")_T = TypeVar("_T")+_T2 = TypeVar("_T2")++# copied from twisted.internet.defer+_SelfResultT = TypeVar("_SelfResultT")+_DeferredListResultItemT = Tuple[bool, _SelfResultT]+DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]]def defer_fail(_failure: Failure) -> Deferred:@@ -62,7 +68,7 @@ def defer_fail(_failure: Failure) -> Deferred:return d-def defer_succeed(result: Any) -> Deferred:+def defer_succeed(result: _T) -> Deferred[_T]:"""Same as twisted.internet.defer.succeed but delay calling callback untilnext reactor loop@@ -128,10 +134,10 @@ def mustbe_deferred(def parallel(iterable: Iterable[_T],count: int,- callable: Callable[Concatenate[_T, _P], Any],+ callable: Callable[Concatenate[_T, _P], _T2],*args: _P.args,**named: _P.kwargs,-) -> Deferred:+) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.@@ -191,12 +197,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):def __init__(self,aiterable: AsyncIterable[_T],- callable: Callable[Concatenate[_T, _P], Any],+ callable: Callable[Concatenate[_T, _P], _T2],*callable_args: _P.args,**callable_kwargs: _P.kwargs,):self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()- self.callable: Callable[Concatenate[_T, _P], Any] = callable+ self.callable: Callable[Concatenate[_T, _P], _T2] = callableself.callable_args: Tuple[Any, ...] = callable_argsself.callable_kwargs: Dict[str, Any] = callable_kwargsself.finished: bool = False@@ -249,10 +255,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):def parallel_async(async_iterable: AsyncIterable[_T],count: int,- callable: Callable[Concatenate[_T, _P], Any],+ callable: Callable[Concatenate[_T, _P], _T2],*args: _P.args,**named: _P.kwargs,-) -> Deferred:+) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:"""Like parallel but for async iterators"""coop = Cooperator()work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 0155c62eb..dd3f8ceb9 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -36,7 +36,7 @@ class ManagerTestCase(TestCase):if not response:response = Response(request.url)- def download_func(**kwargs):+ def download_func(request, spider):return responsedfd = self.mwman.download(download_func, request, self.spider)