Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve typing for Spider.parse(). (#6274)
Andrey Rakhmatullin93f06285309b
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex d6fc0f6c5..ed1629b67 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -18,6 +18,9 @@ jobs:- python-version: 3.8env:TOXENV: typing+ - python-version: 3.8+ env:+ TOXENV: typing-tests- python-version: "3.11" # Keep in sync with .readthedocs.ymlenv:TOXENV: docsdiff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex 7523f3cfe..0c4ebcd23 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider):url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"return [scrapy.Request(url, dont_filter=True)]- def parse(self, response: Response) -> Any: # type: ignore[override]+ def parse(self, response: Response) -> Any:assert isinstance(Response, TextResponse)for link in self.link_extractor.extract_links(response):yield scrapy.Request(link.url, callback=self.parse)diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 72c2aaba7..2416d2a4d 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spiderif TYPE_CHECKING:+ from collections.abc import Callable++ # typing.Concatenate requires Python 3.10# typing.Self requires Python 3.11- from typing_extensions import Self+ from typing_extensions import Concatenate, Selffrom scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings+ CallbackT = Callable[Concatenate[Response, ...], Any]+class Spider(object_ref):"""Base class for scrapy spiders. All spiders must inherit from this@@ -79,10 +84,14 @@ class Spider(object_ref):def _parse(self, response: Response, **kwargs: Any) -> Any:return self.parse(response, **kwargs)- def parse(self, response: Response, **kwargs: Any) -> Any:- raise NotImplementedError(- f"{self.__class__.__name__}.parse callback is not defined"- )+ if TYPE_CHECKING:+ parse: CallbackT+ else:++ def parse(self, response: Response, **kwargs: Any) -> Any:+ raise NotImplementedError(+ f"{self.__class__.__name__}.parse callback is not defined"+ )@classmethoddef update_settings(cls, settings: BaseSettings) -> None:diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testingnew file mode 100644index 000000000..162e31d0c--- /dev/null+++ b/tests_typing/test_spiders.mypy-testing@@ -0,0 +1,68 @@+from typing import Any++import pytest++from scrapy.http import HtmlResponse, Response+from scrapy.spiders import Spider+++class SimpleSpider(Spider):+ pass+++class SameOverrideSpider(Spider):+ def parse(self, response: Response, **kwargs: Any) -> Any:+ pass+++class NoKwargsSpider(Spider):+ def parse(self, response: Response) -> Any:+ pass+++class SpecificKwargsSpider(Spider):+ def parse(self, response: Response, page: int) -> Any:+ pass+++class NarrowOverrideSpider(Spider):+ # without type: ignore this produces several note lines in addition to an error line,+ # which is unsupported by pytest-mypy-testing+ def parse(self, response: HtmlResponse, **kwargs: Any) -> Any: # type: ignore[override]+ pass++[email protected]_testing+def test_spider_parse() -> None:+ spider = Spider()+ reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any++[email protected]_testing+def test_spider_parse_override_simple() -> None:+ spider = SimpleSpider()+ reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any++[email protected]_testing+def test_spider_parse_override_same() -> None:+ spider = SameOverrideSpider()+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any++[email protected]_testing+def test_spider_parse_override_no_kwargs() -> None:+ spider = NoKwargsSpider()+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response) -> Any++[email protected]_testing+def test_spider_parse_override_specific_kwargs() -> None:+ spider = SpecificKwargsSpider()+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any++[email protected]_testing+def test_spider_parse_override_narrow() -> None:+ spider = NarrowOverrideSpider()+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Anydiff --git a/tox.ini b/tox.iniindex d7527bb04..ede139756 100644--- a/tox.ini+++ b/tox.ini@@ -56,6 +56,15 @@ deps =commands =mypy {posargs: scrapy tests}+[testenv:typing-tests]+basepython = python3.8+deps =+ {[test-requirements]deps}+ {[testenv:typing]deps}+ pytest-mypy-testing==0.1.3+commands =+ pytest {posargs: tests_typing}+[testenv:pre-commit]basepython = python3deps =