Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add typing for Crawler.addons.

Andrey Rakhmatullin0258c87dab6d
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 5e7499f9c..fdcc6354d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,7 @@ import logging
import pprint
import signal
import warnings
-from typing import TYPE_CHECKING, Optional, Type, Union
+from typing import TYPE_CHECKING, Optional, Set, Type, Union
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@@ -57,7 +57,7 @@ class Crawler:
spidercls: Type[Spider],
settings: Union[None, dict, Settings] = None,
init_reactor: bool = False,
- addons=None,
+ addons: Optional[AddonManager] = None,
):
if isinstance(spidercls, Spider):
raise ValueError("The spidercls argument must be a class, not an object")
@@ -69,7 +69,7 @@ class Crawler:
self.settings: Settings = settings.copy()
self.spidercls.update_settings(self.settings)
- self.addons = addons if addons is not None else AddonManager()
+ self.addons: AddonManager = addons if addons is not None else AddonManager()
self.addons.load_settings(self.settings)
self.addons.update_addons()
self.addons.check_dependency_clashes()
@@ -199,14 +199,14 @@ class CrawlerRunner:
)
return loader_cls.from_settings(settings.frozencopy())
- def __init__(self, settings=None, addons=None):
+ def __init__(self, settings=None, addons: Optional[AddonManager] = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
- self.addons = addons
+ self.addons: Optional[AddonManager] = addons
self.spider_loader = self._get_spider_loader(settings)
- self._crawlers = set()
- self._active = set()
+ self._crawlers: Set[Crawler] = set()
+ self._active: Set[defer.Deferred] = set()
self.bootstrap_failed = False
@property
@@ -285,7 +285,7 @@ class CrawlerRunner:
def _create_crawler(self, spidercls):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
- return Crawler(spidercls, self.settings, self.addons)
+ return Crawler(spidercls, self.settings, addons=self.addons)
def stop(self):
"""
@@ -331,7 +331,12 @@ class CrawlerProcess(CrawlerRunner):
process. See :ref:`run-from-script` for an example.
"""
- def __init__(self, settings=None, install_root_handler=True, addons=None):
+ def __init__(
+ self,
+ settings=None,
+ install_root_handler: bool = True,
+ addons: Optional[AddonManager] = None,
+ ):
super().__init__(settings, addons)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)