Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix some mypy issues

Matthew Donoughe065db7b56688
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 8570d90bd..2c205a712 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -15,7 +15,7 @@ from scrapy.exceptions import UsageError
class ScrapyCommand:
requires_project = False
- crawler_process = None
+ crawler_process: Any = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
@@ -23,7 +23,7 @@ class ScrapyCommand:
exitcode = 0
def __init__(self):
- self.settings = None # set in scrapy.cmdline
+ self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
if hasattr(self, '_crawler'):
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 01b4a0dbd..facb593da 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -4,7 +4,7 @@ import string
from pathlib import Path
from importlib import import_module
-from typing import Optional
+from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
@@ -116,6 +116,7 @@ class Command(ScrapyCommand):
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
+ return None
def _list_templates(self):
print("Available templates:")
@@ -144,7 +145,7 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
- spiders_dir = Path(spiders_module.__file__).parent
+ spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 9a97e7801..0a9b124b9 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -22,7 +22,7 @@ class Spider(object_ref):
class.
"""
- name: Optional[str] = None
+ name: str
custom_settings: Optional[dict] = None
def __init__(self, name=None, **kwargs):
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 0dfa714e8..82defa033 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -5,7 +5,7 @@ import warnings
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
@@ -176,7 +176,7 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None
'URIs are specified'
)
- result = {}
+ result: Dict[str, Dict[str, Any]] = {}
for element in output:
try:
feed_uri, feed_format = element.rsplit(':', 1)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b61f314ec..f8081f450 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -112,6 +112,7 @@ class ProjectTest(unittest.TestCase):
match = pattern.search(line)
if match is not None:
return match
+ return None
class StartprojectTest(ProjectTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 19f4229a3..a0703ad47 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -292,6 +292,8 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
class ScriptRunnerMixin:
+ script_dir: Path
+
def run_script(self, script_name: str, *script_args):
script_path = self.script_dir / script_name
args = [sys.executable, str(script_path)] + list(script_args)