Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve typing for Spider.parse(). (#6274)

Andrey Rakhmatullin93f06285309b
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d6fc0f6c5..ed1629b67 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -18,6 +18,9 @@ jobs:
- python-version: 3.8
env:
TOXENV: typing
+ - python-version: 3.8
+ env:
+ TOXENV: typing-tests
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 7523f3cfe..0c4ebcd23 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider):
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
return [scrapy.Request(url, dont_filter=True)]
- def parse(self, response: Response) -> Any: # type: ignore[override]
+ def parse(self, response: Response) -> Any:
assert isinstance(Response, TextResponse)
for link in self.link_extractor.extract_links(response):
yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 72c2aaba7..2416d2a4d 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ # typing.Concatenate requires Python 3.10
# typing.Self requires Python 3.11
- from typing_extensions import Self
+ from typing_extensions import Concatenate, Self
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
+ CallbackT = Callable[Concatenate[Response, ...], Any]
+
class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
@@ -79,10 +84,14 @@ class Spider(object_ref):
def _parse(self, response: Response, **kwargs: Any) -> Any:
return self.parse(response, **kwargs)
- def parse(self, response: Response, **kwargs: Any) -> Any:
- raise NotImplementedError(
- f"{self.__class__.__name__}.parse callback is not defined"
- )
+ if TYPE_CHECKING:
+ parse: CallbackT
+ else:
+
+ def parse(self, response: Response, **kwargs: Any) -> Any:
+ raise NotImplementedError(
+ f"{self.__class__.__name__}.parse callback is not defined"
+ )
@classmethod
def update_settings(cls, settings: BaseSettings) -> None:
diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing
new file mode 100644
index 000000000..162e31d0c
--- /dev/null
+++ b/tests_typing/test_spiders.mypy-testing
@@ -0,0 +1,68 @@
+from typing import Any
+
+import pytest
+
+from scrapy.http import HtmlResponse, Response
+from scrapy.spiders import Spider
+
+
+class SimpleSpider(Spider):
+ pass
+
+
+class SameOverrideSpider(Spider):
+ def parse(self, response: Response, **kwargs: Any) -> Any:
+ pass
+
+
+class NoKwargsSpider(Spider):
+ def parse(self, response: Response) -> Any:
+ pass
+
+
+class SpecificKwargsSpider(Spider):
+ def parse(self, response: Response, page: int) -> Any:
+ pass
+
+
+class NarrowOverrideSpider(Spider):
+ # without type: ignore this produces several note lines in addition to an error line,
+ # which is unsupported by pytest-mypy-testing
+ def parse(self, response: HtmlResponse, **kwargs: Any) -> Any: # type: ignore[override]
+ pass
+
+
+def test_spider_parse() -> None:
+ spider = Spider()
+ reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+def test_spider_parse_override_simple() -> None:
+ spider = SimpleSpider()
+ reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+def test_spider_parse_override_same() -> None:
+ spider = SameOverrideSpider()
+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any
+
+
+def test_spider_parse_override_no_kwargs() -> None:
+ spider = NoKwargsSpider()
+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response) -> Any
+
+
+def test_spider_parse_override_specific_kwargs() -> None:
+ spider = SpecificKwargsSpider()
+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+
+
+def test_spider_parse_override_narrow() -> None:
+ spider = NarrowOverrideSpider()
+ reveal_type(spider.parse) # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any
diff --git a/tox.ini b/tox.ini
index d7527bb04..ede139756 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,6 +56,15 @@ deps =
commands =
mypy {posargs: scrapy tests}
+[testenv:typing-tests]
+basepython = python3.8
+deps =
+ {[test-requirements]deps}
+ {[testenv:typing]deps}
+ pytest-mypy-testing==0.1.3
+commands =
+ pytest {posargs: tests_typing}
+
[testenv:pre-commit]
basepython = python3
deps =