Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing for Crawler.addons.
Andrey Rakhmatullin0258c87dab6d
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 5e7499f9c..fdcc6354d 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -4,7 +4,7 @@ import loggingimport pprintimport signalimport warnings-from typing import TYPE_CHECKING, Optional, Type, Union+from typing import TYPE_CHECKING, Optional, Set, Type, Unionfrom twisted.internet import deferfrom zope.interface.exceptions import DoesNotImplement@@ -57,7 +57,7 @@ class Crawler:spidercls: Type[Spider],settings: Union[None, dict, Settings] = None,init_reactor: bool = False,- addons=None,+ addons: Optional[AddonManager] = None,):if isinstance(spidercls, Spider):raise ValueError("The spidercls argument must be a class, not an object")@@ -69,7 +69,7 @@ class Crawler:self.settings: Settings = settings.copy()self.spidercls.update_settings(self.settings)- self.addons = addons if addons is not None else AddonManager()+ self.addons: AddonManager = addons if addons is not None else AddonManager()self.addons.load_settings(self.settings)self.addons.update_addons()self.addons.check_dependency_clashes()@@ -199,14 +199,14 @@ class CrawlerRunner:)return loader_cls.from_settings(settings.frozencopy())- def __init__(self, settings=None, addons=None):+ def __init__(self, settings=None, addons: Optional[AddonManager] = None):if isinstance(settings, dict) or settings is None:settings = Settings(settings)self.settings = settings- self.addons = addons+ self.addons: Optional[AddonManager] = addonsself.spider_loader = self._get_spider_loader(settings)- self._crawlers = set()- self._active = set()+ self._crawlers: Set[Crawler] = set()+ self._active: Set[defer.Deferred] = set()self.bootstrap_failed = False@property@@ -285,7 +285,7 @@ class CrawlerRunner:def _create_crawler(self, spidercls):if isinstance(spidercls, str):spidercls = self.spider_loader.load(spidercls)- return Crawler(spidercls, self.settings, self.addons)+ return Crawler(spidercls, self.settings, addons=self.addons)def stop(self):"""@@ -331,7 +331,12 @@ class CrawlerProcess(CrawlerRunner):process. See :ref:`run-from-script` for an example."""- def __init__(self, settings=None, install_root_handler=True, addons=None):+ def __init__(+ self,+ settings=None,+ install_root_handler: bool = True,+ addons: Optional[AddonManager] = None,+ ):super().__init__(settings, addons)configure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)