Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

More typing for scrapy/utils/defer.py and scrapy/utils/spider.py.

Andrey Rakhmatullinc43798cb9bee
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index ea5a26e77..13d6f9f87 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,11 +1,12 @@
import traceback
import warnings
from collections import defaultdict
+from types import ModuleType
from typing import DefaultDict, Dict, List, Tuple, Type
from zope.interface import implementer
-from scrapy import Spider
+from scrapy import Request, Spider
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import BaseSettings
from scrapy.utils.misc import walk_modules
@@ -45,12 +46,12 @@ class SpiderLoader:
category=UserWarning,
)
- def _load_spiders(self, module):
+ def _load_spiders(self, module: ModuleType) -> None:
for spcls in iter_spider_classes(module):
self._found[spcls.name].append((module.__name__, spcls.__name__))
self._spiders[spcls.name] = spcls
- def _load_all_spiders(self):
+ def _load_all_spiders(self) -> None:
for name in self.spider_modules:
try:
for module in walk_modules(name):
@@ -81,7 +82,7 @@ class SpiderLoader:
except KeyError:
raise KeyError(f"Spider not found: {spider_name}")
- def find_by_request(self, request):
+ def find_by_request(self, request: Request) -> List[str]:
"""
Return the list of spider names that can handle the given request.
"""
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 3502f8b27..388439f4f 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -82,7 +82,7 @@ class Spider(object_ref):
settings.setdict(cls.custom_settings or {}, priority="spider")
@classmethod
- def handles_request(cls, request):
+ def handles_request(cls, request: Request) -> bool:
return url_is_from_spider(request.url, cls)
@staticmethod
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 03f026ce9..bf3c5ef5b 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,11 +5,13 @@ import asyncio
import inspect
from asyncio import Future
from functools import wraps
+from types import CoroutineType
from typing import (
Any,
AsyncGenerator,
AsyncIterable,
AsyncIterator,
+ Awaitable,
Callable,
Coroutine,
Dict,
@@ -19,8 +21,10 @@ from typing import (
List,
Optional,
Tuple,
+ TypeVar,
Union,
cast,
+ overload,
)
from twisted.internet import defer
@@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator):
def _call_anext(self) -> None:
# This starts waiting for the next result from aiterator.
# If aiterator is exhausted, _errback will be called.
- self.anext_deferred = cast(
- Deferred, deferred_from_coro(self.aiterator.__anext__())
- )
+ self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
self.anext_deferred.addCallbacks(self._callback, self._errback)
def __next__(self) -> Deferred:
@@ -297,7 +299,21 @@ async def aiter_errback(
errback(failure.Failure(), *a, **kw)
-def deferred_from_coro(o: Any) -> Any:
+_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
+_T = TypeVar("_T")
+
+
+@overload
+def deferred_from_coro(o: _CT) -> Deferred:
+ ...
+
+
+@overload
+def deferred_from_coro(o: _T) -> _T:
+ ...
+
+
+def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
if isinstance(o, Deferred):
return o
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 86449eeb2..3228eda49 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,14 +1,33 @@
+from __future__ import annotations
+
import inspect
import logging
-
+from types import ModuleType
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ AsyncIterable,
+ Generator,
+ Iterable,
+ Optional,
+ Type,
+ Union,
+)
+
+from twisted.internet.defer import Deferred
+
+from scrapy import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.misc import arg_to_iter
+if TYPE_CHECKING:
+ from scrapy.spiderloader import SpiderLoader
+
logger = logging.getLogger(__name__)
-def iterate_spider_output(result):
+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]:
if inspect.isasyncgen(result):
return result
if inspect.iscoroutine(result):
@@ -18,7 +37,7 @@ def iterate_spider_output(result):
return arg_to_iter(deferred_from_coro(result))
-def iter_spider_classes(module):
+def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
"""Return an iterator over all spider classes defined in the given module
that can be instantiated (i.e. which have name)
"""
@@ -37,8 +56,12 @@ def iter_spider_classes(module):
def spidercls_for_request(
- spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False
-):
+ spider_loader: SpiderLoader,
+ request: Request,
+ default_spidercls: Optional[Type[Spider]] = None,
+ log_none: bool = False,
+ log_multiple: bool = False,
+) -> Optional[Type[Spider]]:
"""Return a spider class that handles the given Request.
This will look for the spiders that can handle the given request (using