Scrapy, a fast high-level web crawling & scraping framework for Python.
More typing for scrapy/utils/defer.py and scrapy/utils/spider.py.
Andrey Rakhmatullinc43798cb9bee
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex ea5a26e77..13d6f9f87 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -1,11 +1,12 @@import tracebackimport warningsfrom collections import defaultdict+from types import ModuleTypefrom typing import DefaultDict, Dict, List, Tuple, Typefrom zope.interface import implementer-from scrapy import Spider+from scrapy import Request, Spiderfrom scrapy.interfaces import ISpiderLoaderfrom scrapy.settings import BaseSettingsfrom scrapy.utils.misc import walk_modules@@ -45,12 +46,12 @@ class SpiderLoader:category=UserWarning,)- def _load_spiders(self, module):+ def _load_spiders(self, module: ModuleType) -> None:for spcls in iter_spider_classes(module):self._found[spcls.name].append((module.__name__, spcls.__name__))self._spiders[spcls.name] = spcls- def _load_all_spiders(self):+ def _load_all_spiders(self) -> None:for name in self.spider_modules:try:for module in walk_modules(name):@@ -81,7 +82,7 @@ class SpiderLoader:except KeyError:raise KeyError(f"Spider not found: {spider_name}")- def find_by_request(self, request):+ def find_by_request(self, request: Request) -> List[str]:"""Return the list of spider names that can handle the given request."""diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 3502f8b27..388439f4f 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -82,7 +82,7 @@ class Spider(object_ref):settings.setdict(cls.custom_settings or {}, priority="spider")@classmethod- def handles_request(cls, request):+ def handles_request(cls, request: Request) -> bool:return url_is_from_spider(request.url, cls)@staticmethoddiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 03f026ce9..bf3c5ef5b 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -5,11 +5,13 @@ import asyncioimport inspectfrom asyncio import Futurefrom functools import wraps+from types import CoroutineTypefrom typing import (Any,AsyncGenerator,AsyncIterable,AsyncIterator,+ Awaitable,Callable,Coroutine,Dict,@@ -19,8 +21,10 @@ from typing import (List,Optional,Tuple,+ TypeVar,Union,cast,+ overload,)from twisted.internet import defer@@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator):def _call_anext(self) -> None:# This starts waiting for the next result from aiterator.# If aiterator is exhausted, _errback will be called.- self.anext_deferred = cast(- Deferred, deferred_from_coro(self.aiterator.__anext__())- )+ self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())self.anext_deferred.addCallbacks(self._callback, self._errback)def __next__(self) -> Deferred:@@ -297,7 +299,21 @@ async def aiter_errback(errback(failure.Failure(), *a, **kw)-def deferred_from_coro(o: Any) -> Any:+_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])+_T = TypeVar("_T")+++@overload+def deferred_from_coro(o: _CT) -> Deferred:+ ...+++@overload+def deferred_from_coro(o: _T) -> _T:+ ...+++def deferred_from_coro(o: _T) -> Union[Deferred, _T]:"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""if isinstance(o, Deferred):return odiff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 86449eeb2..3228eda49 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -1,14 +1,33 @@+from __future__ import annotations+import inspectimport logging-+from types import ModuleType+from typing import (+ TYPE_CHECKING,+ Any,+ AsyncIterable,+ Generator,+ Iterable,+ Optional,+ Type,+ Union,+)++from twisted.internet.defer import Deferred++from scrapy import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.misc import arg_to_iter+if TYPE_CHECKING:+ from scrapy.spiderloader import SpiderLoader+logger = logging.getLogger(__name__)-def iterate_spider_output(result):+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]:if inspect.isasyncgen(result):return resultif inspect.iscoroutine(result):@@ -18,7 +37,7 @@ def iterate_spider_output(result):return arg_to_iter(deferred_from_coro(result))-def iter_spider_classes(module):+def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:"""Return an iterator over all spider classes defined in the given modulethat can be instantiated (i.e. which have name)"""@@ -37,8 +56,12 @@ def iter_spider_classes(module):def spidercls_for_request(- spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False-):+ spider_loader: SpiderLoader,+ request: Request,+ default_spidercls: Optional[Type[Spider]] = None,+ log_none: bool = False,+ log_multiple: bool = False,+) -> Optional[Type[Spider]]:"""Return a spider class that handles the given Request.This will look for the spiders that can handle the given request (using