Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

change crawler_process to optional

Matthew Donoughe726680c7125a
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 2c205a712..1f088041c 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -4,9 +4,10 @@ Base class for Scrapy commands
import os
import argparse
from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, Optional
from twisted.python import failure
+from scrapy.crawler import CrawlerProcess
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
@@ -15,7 +16,7 @@ from scrapy.exceptions import UsageError
class ScrapyCommand:
requires_project = False
- crawler_process: Any = None
+ crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index facb593da..ed9660252 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -133,6 +133,10 @@ class Command(ScrapyCommand):
return True
return False
+ assert (
+ self.crawler_process is not None
+ ), "crawler_process must be set before calling run"
+
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError: