Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use CallbackT for Request.callback. (#6422)

Andrey Rakhmatullin558b1d11d2f1
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1265aa38e..fbd200d88 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -9,7 +9,6 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
- Callable,
Coroutine,
Dict,
Iterable,
@@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request
if TYPE_CHECKING:
from twisted.python.failure import Failure
+ from scrapy.http.request import CallbackT
from scrapy.spiders import Spider
@@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand):
opts: argparse.Namespace,
depth: int,
spider: Spider,
- callback: Callable,
- ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
+ callback: CallbackT,
+ ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]:
items, requests = [], []
for x in spider_output:
if is_item(x):
@@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand):
def run_callback(
self,
response: Response,
- callback: Callable,
+ callback: CallbackT,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> Deferred[Any]:
cb_kwargs = cb_kwargs or {}
@@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand):
def get_callback_from_rules(
self, spider: Spider, response: Response
- ) -> Union[Callable, str, None]:
+ ) -> Union[CallbackT, str, None]:
if getattr(spider, "rules", None):
for rule in spider.rules: # type: ignore[attr-defined]
if rule.link_extractor.matches(response.url):
@@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand):
def scraped_data(
self,
args: Tuple[
- List[Any], List[Request], argparse.Namespace, int, Spider, Callable
+ List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT
],
) -> List[Any]:
items, requests, opts, depth, spider, callback = args
@@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand):
spider: Spider,
opts: argparse.Namespace,
response: Optional[Response] = None,
- ) -> Callable:
- cb: Union[str, Callable, None] = None
+ ) -> CallbackT:
+ cb: Union[str, CallbackT, None] = None
if response:
cb = response.meta["_callback"]
if not cb:
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 440e0dc44..a7e129948 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -16,6 +16,7 @@ from typing import (
Optional,
Tuple,
Type,
+ cast,
)
from unittest import TestCase, TestResult
@@ -62,7 +63,7 @@ class Contract:
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list( # pylint: disable=return-in-finally
- iterate_spider_output(cb_result)
+ cast(Iterable[Any], iterate_spider_output(cb_result))
)
request.callback = wrapper
@@ -79,7 +80,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
- output = list(iterate_spider_output(cb_result))
+ output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@@ -195,7 +196,7 @@ class ContractsManager:
def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
try:
output = cb(response, **cb_kwargs)
- output = list(iterate_spider_output(output))
+ output = list(cast(Iterable[Any], iterate_spider_output(output)))
except Exception:
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 04589dd37..9381a6cb3 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
AnyStr,
- Callable,
Dict,
Iterable,
List,
@@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from twisted.python.failure import Failure
+
+ # typing.Concatenate requires Python 3.10
# typing.NotRequired and typing.Self require Python 3.11
- from typing_extensions import NotRequired, Self
+ from typing_extensions import Concatenate, NotRequired, Self
+
+ from scrapy.http import Response
+
+ CallbackT = Callable[Concatenate[Response, ...], Any]
class VerboseCookie(TypedDict):
@@ -110,7 +118,7 @@ class Request(object_ref):
def __init__(
self,
url: str,
- callback: Optional[Callable] = None,
+ callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@@ -119,7 +127,7 @@ class Request(object_ref):
encoding: str = "utf-8",
priority: int = 0,
dont_filter: bool = False,
- errback: Optional[Callable] = None,
+ errback: Optional[Callable[[Failure], Any]] = None,
flags: Optional[List[str]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
@@ -137,8 +145,8 @@ class Request(object_ref):
)
if not (callable(errback) or errback is None):
raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
- self.callback: Optional[Callable] = callback
- self.errback: Optional[Callable] = errback
+ self.callback: Optional[CallbackT] = callback
+ self.errback: Optional[Callable[[Failure], Any]] = errback
self.cookies: CookiesT = cookies or {}
self.headers: Headers = Headers(headers or {}, encoding=encoding)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index ff3581abb..92e4852b6 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -27,7 +27,7 @@ from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
-from scrapy.http.request import CookiesT, Request
+from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
@@ -35,10 +35,12 @@ if TYPE_CHECKING:
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.ssl import Certificate
+ from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.http.request import CallbackT, CookiesT
from scrapy.selector import SelectorList
@@ -196,7 +198,7 @@ class Response(object_ref):
def follow(
self,
url: Union[str, Link],
- callback: Optional[Callable] = None,
+ callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@@ -205,7 +207,7 @@ class Response(object_ref):
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
- errback: Optional[Callable] = None,
+ errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@@ -249,7 +251,7 @@ class Response(object_ref):
def follow_all(
self,
urls: Iterable[Union[str, Link]],
- callback: Optional[Callable] = None,
+ callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@@ -258,7 +260,7 @@ class Response(object_ref):
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
- errback: Optional[Callable] = None,
+ errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Iterable[Request]:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0635f744f..588695002 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
- from scrapy.http.request import CookiesT, Request
+ from twisted.python.failure import Failure
+
+ from scrapy.http.request import CallbackT, CookiesT, Request
from scrapy.selector import Selector, SelectorList
@@ -179,7 +181,7 @@ class TextResponse(Response):
def follow(
self,
url: Union[str, Link, parsel.Selector],
- callback: Optional[Callable] = None,
+ callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@@ -188,7 +190,7 @@ class TextResponse(Response):
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
- errback: Optional[Callable] = None,
+ errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@@ -232,7 +234,7 @@ class TextResponse(Response):
def follow_all(
self,
urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,
- callback: Optional[Callable] = None,
+ callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@@ -241,7 +243,7 @@ class TextResponse(Response):
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
- errback: Optional[Callable] = None,
+ errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
css: Optional[str] = None,
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index f0b0c0988..d977acd26 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
- from collections.abc import Callable
-
from twisted.internet.defer import Deferred
- # typing.Concatenate requires Python 3.10
# typing.Self requires Python 3.11
- from typing_extensions import Concatenate, Self
+ from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.http.request import CallbackT
from scrapy.settings import BaseSettings, _SettingsKeyT
from scrapy.utils.log import SpiderLoggerAdapter
- CallbackT = Callable[Concatenate[Response, ...], Any]
-
class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 48c830d2a..2639f14b2 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.http.request import CallbackT
_T = TypeVar("_T")
@@ -73,7 +74,7 @@ class Rule:
def __init__(
self,
link_extractor: Optional[LinkExtractor] = None,
- callback: Union[Callable, str, None] = None,
+ callback: Union[CallbackT, str, None] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
follow: Optional[bool] = None,
process_links: Union[ProcessLinksT, str, None] = None,
@@ -81,7 +82,7 @@ class Rule:
errback: Union[Callable[[Failure], Any], str, None] = None,
):
self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor
- self.callback: Union[Callable, str, None] = callback
+ self.callback: Union[CallbackT, str, None] = callback
self.errback: Union[Callable[[Failure], Any], str, None] = errback
self.cb_kwargs: Dict[str, Any] = cb_kwargs or {}
self.process_links: Union[ProcessLinksT, str] = process_links or _identity
@@ -92,7 +93,7 @@ class Rule:
def _compile(self, spider: Spider) -> None:
# this replaces method names with methods and we can't express this in type hints
- self.callback = _get_method(self.callback, spider)
+ self.callback = cast("CallbackT", _get_method(self.callback, spider))
self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
self.process_links = cast(
ProcessLinksT, _get_method(self.process_links, spider)
@@ -122,7 +123,9 @@ class CrawlSpider(Spider):
def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
return []
- def process_results(self, response: Response, results: Any) -> Any:
+ def process_results(
+ self, response: Response, results: Iterable[Any]
+ ) -> Iterable[Any]:
return results
def _build_request(self, rule_index: int, link: Link) -> Request:
@@ -152,7 +155,7 @@ class CrawlSpider(Spider):
rule = self._rules[cast(int, response.meta["rule"])]
return self._parse_response(
response,
- cast(Callable, rule.callback),
+ cast("CallbackT", rule.callback),
{**rule.cb_kwargs, **cb_kwargs},
rule.follow,
)
@@ -166,7 +169,7 @@ class CrawlSpider(Spider):
async def _parse_response(
self,
response: Response,
- callback: Optional[Callable],
+ callback: Optional[CallbackT],
cb_kwargs: Dict[str, Any],
follow: bool = True,
) -> AsyncIterable[Any]:
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d082fbfdb..1542ef79c 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -5,7 +5,6 @@ import re
from typing import (
TYPE_CHECKING,
Any,
- Callable,
Dict,
Iterable,
List,
@@ -27,6 +26,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.http.request import CallbackT
logger = logging.getLogger(__name__)
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
class SitemapSpider(Spider):
sitemap_urls: Sequence[str] = ()
sitemap_rules: Sequence[
- Tuple[Union[re.Pattern[str], str], Union[str, Callable]]
+ Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]]
] = [("", "parse")]
sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""]
sitemap_alternate_links: bool = False
@@ -54,10 +54,10 @@ class SitemapSpider(Spider):
def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
- self._cbs: List[Tuple[re.Pattern[str], Callable]] = []
+ self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = []
for r, c in self.sitemap_rules:
if isinstance(c, str):
- c = cast(Callable, getattr(self, c))
+ c = cast("CallbackT", getattr(self, c))
self._cbs.append((regex(r), c))
self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]