Scrapy, a fast high-level web crawling & scraping framework for Python.
Use CallbackT for Request.callback. (#6422)
Andrey Rakhmatullin558b1d11d2f1
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 1265aa38e..fbd200d88 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -9,7 +9,6 @@ from typing import (TYPE_CHECKING,Any,AsyncGenerator,- Callable,Coroutine,Dict,Iterable,@@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_requestif TYPE_CHECKING:from twisted.python.failure import Failure+ from scrapy.http.request import CallbackTfrom scrapy.spiders import Spider@@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand):opts: argparse.Namespace,depth: int,spider: Spider,- callback: Callable,- ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:+ callback: CallbackT,+ ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]:items, requests = [], []for x in spider_output:if is_item(x):@@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand):def run_callback(self,response: Response,- callback: Callable,+ callback: CallbackT,cb_kwargs: Optional[Dict[str, Any]] = None,) -> Deferred[Any]:cb_kwargs = cb_kwargs or {}@@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand):def get_callback_from_rules(self, spider: Spider, response: Response- ) -> Union[Callable, str, None]:+ ) -> Union[CallbackT, str, None]:if getattr(spider, "rules", None):for rule in spider.rules: # type: ignore[attr-defined]if rule.link_extractor.matches(response.url):@@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand):def scraped_data(self,args: Tuple[- List[Any], List[Request], argparse.Namespace, int, Spider, Callable+ List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT],) -> List[Any]:items, requests, opts, depth, spider, callback = args@@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand):spider: Spider,opts: argparse.Namespace,response: Optional[Response] = None,- ) -> Callable:- cb: Union[str, Callable, None] = None+ ) -> CallbackT:+ cb: Union[str, CallbackT, None] = Noneif response:cb = response.meta["_callback"]if not cb:diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 440e0dc44..a7e129948 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -16,6 +16,7 @@ from typing import (Optional,Tuple,Type,+ cast,)from unittest import TestCase, TestResult@@ -62,7 +63,7 @@ class Contract:if isinstance(cb_result, (AsyncGenerator, CoroutineType)):raise TypeError("Contracts don't support async callbacks")return list( # pylint: disable=return-in-finally- iterate_spider_output(cb_result)+ cast(Iterable[Any], iterate_spider_output(cb_result)))request.callback = wrapper@@ -79,7 +80,7 @@ class Contract:cb_result = cb(response, **cb_kwargs)if isinstance(cb_result, (AsyncGenerator, CoroutineType)):raise TypeError("Contracts don't support async callbacks")- output = list(iterate_spider_output(cb_result))+ output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))try:results.startTest(self.testcase_post)self.post_process(output)@@ -195,7 +196,7 @@ class ContractsManager:def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:try:output = cb(response, **cb_kwargs)- output = list(iterate_spider_output(output))+ output = list(cast(Iterable[Any], iterate_spider_output(output)))except Exception:case = _create_testcase(method, "callback")results.addError(case, sys.exc_info())diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 04589dd37..9381a6cb3 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -12,7 +12,6 @@ from typing import (TYPE_CHECKING,Any,AnyStr,- Callable,Dict,Iterable,List,@@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_reffrom scrapy.utils.url import escape_ajaxif TYPE_CHECKING:+ from collections.abc import Callable++ from twisted.python.failure import Failure++ # typing.Concatenate requires Python 3.10# typing.NotRequired and typing.Self require Python 3.11- from typing_extensions import NotRequired, Self+ from typing_extensions import Concatenate, NotRequired, Self++ from scrapy.http import Response++ CallbackT = Callable[Concatenate[Response, ...], Any]class VerboseCookie(TypedDict):@@ -110,7 +118,7 @@ class Request(object_ref):def __init__(self,url: str,- callback: Optional[Callable] = None,+ callback: Optional[CallbackT] = None,method: str = "GET",headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,body: Optional[Union[bytes, str]] = None,@@ -119,7 +127,7 @@ class Request(object_ref):encoding: str = "utf-8",priority: int = 0,dont_filter: bool = False,- errback: Optional[Callable] = None,+ errback: Optional[Callable[[Failure], Any]] = None,flags: Optional[List[str]] = None,cb_kwargs: Optional[Dict[str, Any]] = None,) -> None:@@ -137,8 +145,8 @@ class Request(object_ref):)if not (callable(errback) or errback is None):raise TypeError(f"errback must be a callable, got {type(errback).__name__}")- self.callback: Optional[Callable] = callback- self.errback: Optional[Callable] = errback+ self.callback: Optional[CallbackT] = callback+ self.errback: Optional[Callable[[Failure], Any]] = errbackself.cookies: CookiesT = cookies or {}self.headers: Headers = Headers(headers or {}, encoding=encoding)diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex ff3581abb..92e4852b6 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -27,7 +27,7 @@ from urllib.parse import urljoinfrom scrapy.exceptions import NotSupportedfrom scrapy.http.headers import Headers-from scrapy.http.request import CookiesT, Request+from scrapy.http.request import Requestfrom scrapy.link import Linkfrom scrapy.utils.trackref import object_ref@@ -35,10 +35,12 @@ if TYPE_CHECKING:from ipaddress import IPv4Address, IPv6Addressfrom twisted.internet.ssl import Certificate+ from twisted.python.failure import Failure# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.http.request import CallbackT, CookiesTfrom scrapy.selector import SelectorList@@ -196,7 +198,7 @@ class Response(object_ref):def follow(self,url: Union[str, Link],- callback: Optional[Callable] = None,+ callback: Optional[CallbackT] = None,method: str = "GET",headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,body: Optional[Union[bytes, str]] = None,@@ -205,7 +207,7 @@ class Response(object_ref):encoding: Optional[str] = "utf-8",priority: int = 0,dont_filter: bool = False,- errback: Optional[Callable] = None,+ errback: Optional[Callable[[Failure], Any]] = None,cb_kwargs: Optional[Dict[str, Any]] = None,flags: Optional[List[str]] = None,) -> Request:@@ -249,7 +251,7 @@ class Response(object_ref):def follow_all(self,urls: Iterable[Union[str, Link]],- callback: Optional[Callable] = None,+ callback: Optional[CallbackT] = None,method: str = "GET",headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,body: Optional[Union[bytes, str]] = None,@@ -258,7 +260,7 @@ class Response(object_ref):encoding: Optional[str] = "utf-8",priority: int = 0,dont_filter: bool = False,- errback: Optional[Callable] = None,+ errback: Optional[Callable[[Failure], Any]] = None,cb_kwargs: Optional[Dict[str, Any]] = None,flags: Optional[List[str]] = None,) -> Iterable[Request]:diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 0635f744f..588695002 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicodefrom scrapy.utils.response import get_base_urlif TYPE_CHECKING:- from scrapy.http.request import CookiesT, Request+ from twisted.python.failure import Failure++ from scrapy.http.request import CallbackT, CookiesT, Requestfrom scrapy.selector import Selector, SelectorList@@ -179,7 +181,7 @@ class TextResponse(Response):def follow(self,url: Union[str, Link, parsel.Selector],- callback: Optional[Callable] = None,+ callback: Optional[CallbackT] = None,method: str = "GET",headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,body: Optional[Union[bytes, str]] = None,@@ -188,7 +190,7 @@ class TextResponse(Response):encoding: Optional[str] = None,priority: int = 0,dont_filter: bool = False,- errback: Optional[Callable] = None,+ errback: Optional[Callable[[Failure], Any]] = None,cb_kwargs: Optional[Dict[str, Any]] = None,flags: Optional[List[str]] = None,) -> Request:@@ -232,7 +234,7 @@ class TextResponse(Response):def follow_all(self,urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,- callback: Optional[Callable] = None,+ callback: Optional[CallbackT] = None,method: str = "GET",headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,body: Optional[Union[bytes, str]] = None,@@ -241,7 +243,7 @@ class TextResponse(Response):encoding: Optional[str] = None,priority: int = 0,dont_filter: bool = False,- errback: Optional[Callable] = None,+ errback: Optional[Callable[[Failure], Any]] = None,cb_kwargs: Optional[Dict[str, Any]] = None,flags: Optional[List[str]] = None,css: Optional[str] = None,diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex f0b0c0988..d977acd26 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spiderif TYPE_CHECKING:- from collections.abc import Callable-from twisted.internet.defer import Deferred- # typing.Concatenate requires Python 3.10# typing.Self requires Python 3.11- from typing_extensions import Concatenate, Self+ from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.http.request import CallbackTfrom scrapy.settings import BaseSettings, _SettingsKeyTfrom scrapy.utils.log import SpiderLoggerAdapter- CallbackT = Callable[Concatenate[Response, ...], Any]-class Spider(object_ref):"""Base class for scrapy spiders. All spiders must inherit from thisdiff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 48c830d2a..2639f14b2 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -39,6 +39,7 @@ if TYPE_CHECKING:from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.http.request import CallbackT_T = TypeVar("_T")@@ -73,7 +74,7 @@ class Rule:def __init__(self,link_extractor: Optional[LinkExtractor] = None,- callback: Union[Callable, str, None] = None,+ callback: Union[CallbackT, str, None] = None,cb_kwargs: Optional[Dict[str, Any]] = None,follow: Optional[bool] = None,process_links: Union[ProcessLinksT, str, None] = None,@@ -81,7 +82,7 @@ class Rule:errback: Union[Callable[[Failure], Any], str, None] = None,):self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor- self.callback: Union[Callable, str, None] = callback+ self.callback: Union[CallbackT, str, None] = callbackself.errback: Union[Callable[[Failure], Any], str, None] = errbackself.cb_kwargs: Dict[str, Any] = cb_kwargs or {}self.process_links: Union[ProcessLinksT, str] = process_links or _identity@@ -92,7 +93,7 @@ class Rule:def _compile(self, spider: Spider) -> None:# this replaces method names with methods and we can't express this in type hints- self.callback = _get_method(self.callback, spider)+ self.callback = cast("CallbackT", _get_method(self.callback, spider))self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))self.process_links = cast(ProcessLinksT, _get_method(self.process_links, spider)@@ -122,7 +123,9 @@ class CrawlSpider(Spider):def parse_start_url(self, response: Response, **kwargs: Any) -> Any:return []- def process_results(self, response: Response, results: Any) -> Any:+ def process_results(+ self, response: Response, results: Iterable[Any]+ ) -> Iterable[Any]:return resultsdef _build_request(self, rule_index: int, link: Link) -> Request:@@ -152,7 +155,7 @@ class CrawlSpider(Spider):rule = self._rules[cast(int, response.meta["rule"])]return self._parse_response(response,- cast(Callable, rule.callback),+ cast("CallbackT", rule.callback),{**rule.cb_kwargs, **cb_kwargs},rule.follow,)@@ -166,7 +169,7 @@ class CrawlSpider(Spider):async def _parse_response(self,response: Response,- callback: Optional[Callable],+ callback: Optional[CallbackT],cb_kwargs: Dict[str, Any],follow: bool = True,) -> AsyncIterable[Any]:diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex d082fbfdb..1542ef79c 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -5,7 +5,6 @@ import refrom typing import (TYPE_CHECKING,Any,- Callable,Dict,Iterable,List,@@ -27,6 +26,7 @@ if TYPE_CHECKING:from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.http.request import CallbackTlogger = logging.getLogger(__name__)@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)class SitemapSpider(Spider):sitemap_urls: Sequence[str] = ()sitemap_rules: Sequence[- Tuple[Union[re.Pattern[str], str], Union[str, Callable]]+ Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]]] = [("", "parse")]sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""]sitemap_alternate_links: bool = False@@ -54,10 +54,10 @@ class SitemapSpider(Spider):def __init__(self, *a: Any, **kw: Any):super().__init__(*a, **kw)- self._cbs: List[Tuple[re.Pattern[str], Callable]] = []+ self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = []for r, c in self.sitemap_rules:if isinstance(c, str):- c = cast(Callable, getattr(self, c))+ c = cast("CallbackT", getattr(self, c))self._cbs.append((regex(r), c))self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]