Scrapy, a fast high-level web crawling & scraping framework for Python.
fix some mypy issues
Matthew Donoughe065db7b56688
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 8570d90bd..2c205a712 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -15,7 +15,7 @@ from scrapy.exceptions import UsageErrorclass ScrapyCommand:requires_project = False- crawler_process = None+ crawler_process: Any = None# default settings to be used for this command instead of global defaultsdefault_settings: Dict[str, Any] = {}@@ -23,7 +23,7 @@ class ScrapyCommand:exitcode = 0def __init__(self):- self.settings = None # set in scrapy.cmdline+ self.settings: Any = None # set in scrapy.cmdlinedef set_crawler(self, crawler):if hasattr(self, '_crawler'):diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 01b4a0dbd..facb593da 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -4,7 +4,7 @@ import stringfrom pathlib import Pathfrom importlib import import_module-from typing import Optional+from typing import Optional, castfrom urllib.parse import urlparseimport scrapy@@ -116,6 +116,7 @@ class Command(ScrapyCommand):return template_fileprint(f"Unable to find template: {template}\n")print('Use "scrapy genspider --list" to see all available templates.')+ return Nonedef _list_templates(self):print("Available templates:")@@ -144,7 +145,7 @@ class Command(ScrapyCommand):# a file with the same name exists in the target directoryspiders_module = import_module(self.settings['NEWSPIDER_MODULE'])- spiders_dir = Path(spiders_module.__file__).parent+ spiders_dir = Path(cast(str, spiders_module.__file__)).parentspiders_dir_abs = spiders_dir.resolve()path = spiders_dir_abs / (name + ".py")if path.exists():diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 9a97e7801..0a9b124b9 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -22,7 +22,7 @@ class Spider(object_ref):class."""- name: Optional[str] = None+ name: strcustom_settings: Optional[dict] = Nonedef __init__(self, name=None, **kwargs):diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 0dfa714e8..82defa033 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -5,7 +5,7 @@ import warningsfrom configparser import ConfigParserfrom operator import itemgetterfrom pathlib import Path-from typing import List, Optional, Union+from typing import Any, Dict, List, Optional, Unionfrom scrapy.exceptions import ScrapyDeprecationWarning, UsageError@@ -176,7 +176,7 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None'URIs are specified')- result = {}+ result: Dict[str, Dict[str, Any]] = {}for element in output:try:feed_uri, feed_format = element.rsplit(':', 1)diff --git a/tests/test_commands.py b/tests/test_commands.pyindex b61f314ec..f8081f450 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -112,6 +112,7 @@ class ProjectTest(unittest.TestCase):match = pattern.search(line)if match is not None:return match+ return Noneclass StartprojectTest(ProjectTest):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 19f4229a3..a0703ad47 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -292,6 +292,8 @@ class CrawlerRunnerHasSpider(unittest.TestCase):class ScriptRunnerMixin:+ script_dir: Path+def run_script(self, script_name: str, *script_args):script_path = self.script_dir / script_nameargs = [sys.executable, str(script_path)] + list(script_args)