Scrapy, a fast high-level web crawling & scraping framework for Python.
Typing: annotate a few Spider attributes
Eugenio Lacuestaa8114d3731cf
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 12b4fba09..a66d65846 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst"""import loggingimport warnings+from typing import Optionalfrom scrapy import signalsfrom scrapy.http import Request@@ -18,8 +19,8 @@ class Spider(object_ref):class."""- name = None- custom_settings = None+ name: Optional[str] = None+ custom_settings: Optional[dict] = Nonedef __init__(self, name=None, **kwargs):if name is not None:diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex c9fbce08d..bc4551a54 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rstimport copyimport warnings+from typing import Sequencefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, HtmlResponse@@ -72,7 +73,7 @@ class Rule:class CrawlSpider(Spider):- rules = ()+ rules: Sequence[Rule] = ()def __init__(self, *a, **kw):super().__init__(*a, **kw)diff --git a/setup.cfg b/setup.cfgindex 3a624ec94..8101443e3 100644--- a/setup.cfg+++ b/setup.cfg@@ -16,9 +16,6 @@ ignore_errors = True[mypy-scrapy.commands]ignore_errors = True-[mypy-scrapy.commands.bench]-ignore_errors = True-[mypy-scrapy.commands.parse]ignore_errors = True@@ -28,9 +25,6 @@ ignore_errors = True[mypy-scrapy.contracts]ignore_errors = True-[mypy-scrapy.core.spidermw]-ignore_errors = True-[mypy-scrapy.interfaces]ignore_errors = True@@ -70,15 +64,6 @@ ignore_errors = True[mypy-tests.mocks.dummydbm]ignore_errors = True-[mypy-tests.spiders]-ignore_errors = True--[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]-ignore_errors = True--[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]-ignore_errors = True-[mypy-tests.test_command_fetch]ignore_errors = True@@ -94,9 +79,6 @@ ignore_errors = True[mypy-tests.test_contracts]ignore_errors = True-[mypy-tests.test_crawler]-ignore_errors = True-[mypy-tests.test_downloader_handlers]ignore_errors = True@@ -127,53 +109,20 @@ ignore_errors = True[mypy-tests.test_pipeline_images]ignore_errors = True-[mypy-tests.test_pipelines]-ignore_errors = True--[mypy-tests.test_request_attribute_binding]-ignore_errors = True-[mypy-tests.test_request_cb_kwargs]ignore_errors = True-[mypy-tests.test_request_left]-ignore_errors = True-[mypy-tests.test_scheduler]ignore_errors = True-[mypy-tests.test_signals]-ignore_errors = True--[mypy-tests.test_spiderloader.test_spiders.nested.spider4]-ignore_errors = True--[mypy-tests.test_spiderloader.test_spiders.spider1]-ignore_errors = True--[mypy-tests.test_spiderloader.test_spiders.spider2]-ignore_errors = True--[mypy-tests.test_spiderloader.test_spiders.spider3]-ignore_errors = True-[mypy-tests.test_spidermiddleware_httperror]ignore_errors = True-[mypy-tests.test_spidermiddleware_output_chain]-ignore_errors = True-[mypy-tests.test_spidermiddleware_referer]ignore_errors = True-[mypy-tests.test_utils_reqser]-ignore_errors = True-[mypy-tests.test_utils_serialize]ignore_errors = True-[mypy-tests.test_utils_spider]-ignore_errors = True-[mypy-tests.test_utils_url]ignore_errors = Truediff --git a/tests/spiders.py b/tests/spiders.pyindex 63bd726fb..8a0ee44b7 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):A CrawlSpider which overrides the 'parse' method"""name = 'crawl_spider_with_parse_method'- custom_settings = {+ custom_settings: dict = {'RETRY_HTTP_CODES': [], # no need to retry}rules = (