Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Typing: annotate a few Spider attributes

Eugenio Lacuestaa8114d3731cf
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 12b4fba09..a66d65846 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst
"""
import logging
import warnings
+from typing import Optional
from scrapy import signals
from scrapy.http import Request
@@ -18,8 +19,8 @@ class Spider(object_ref):
class.
"""
- name = None
- custom_settings = None
+ name: Optional[str] = None
+ custom_settings: Optional[dict] = None
def __init__(self, name=None, **kwargs):
if name is not None:
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index c9fbce08d..bc4551a54 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst
import copy
import warnings
+from typing import Sequence
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
@@ -72,7 +73,7 @@ class Rule:
class CrawlSpider(Spider):
- rules = ()
+ rules: Sequence[Rule] = ()
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
diff --git a/setup.cfg b/setup.cfg
index 3a624ec94..8101443e3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,9 +16,6 @@ ignore_errors = True
[mypy-scrapy.commands]
ignore_errors = True
-[mypy-scrapy.commands.bench]
-ignore_errors = True
-
[mypy-scrapy.commands.parse]
ignore_errors = True
@@ -28,9 +25,6 @@ ignore_errors = True
[mypy-scrapy.contracts]
ignore_errors = True
-[mypy-scrapy.core.spidermw]
-ignore_errors = True
-
[mypy-scrapy.interfaces]
ignore_errors = True
@@ -70,15 +64,6 @@ ignore_errors = True
[mypy-tests.mocks.dummydbm]
ignore_errors = True
-[mypy-tests.spiders]
-ignore_errors = True
-
-[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
-ignore_errors = True
-
-[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
-ignore_errors = True
-
[mypy-tests.test_command_fetch]
ignore_errors = True
@@ -94,9 +79,6 @@ ignore_errors = True
[mypy-tests.test_contracts]
ignore_errors = True
-[mypy-tests.test_crawler]
-ignore_errors = True
-
[mypy-tests.test_downloader_handlers]
ignore_errors = True
@@ -127,53 +109,20 @@ ignore_errors = True
[mypy-tests.test_pipeline_images]
ignore_errors = True
-[mypy-tests.test_pipelines]
-ignore_errors = True
-
-[mypy-tests.test_request_attribute_binding]
-ignore_errors = True
-
[mypy-tests.test_request_cb_kwargs]
ignore_errors = True
-[mypy-tests.test_request_left]
-ignore_errors = True
-
[mypy-tests.test_scheduler]
ignore_errors = True
-[mypy-tests.test_signals]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider1]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider2]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider3]
-ignore_errors = True
-
[mypy-tests.test_spidermiddleware_httperror]
ignore_errors = True
-[mypy-tests.test_spidermiddleware_output_chain]
-ignore_errors = True
-
[mypy-tests.test_spidermiddleware_referer]
ignore_errors = True
-[mypy-tests.test_utils_reqser]
-ignore_errors = True
-
[mypy-tests.test_utils_serialize]
ignore_errors = True
-[mypy-tests.test_utils_spider]
-ignore_errors = True
-
[mypy-tests.test_utils_url]
ignore_errors = True
diff --git a/tests/spiders.py b/tests/spiders.py
index 63bd726fb..8a0ee44b7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
A CrawlSpider which overrides the 'parse' method
"""
name = 'crawl_spider_with_parse_method'
- custom_settings = {
+ custom_settings: dict = {
'RETRY_HTTP_CODES': [], # no need to retry
}
rules = (