Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'origin/master' into addons
Andrey Rakhmatullin4e21e3e59df6
diff --cc scrapy/cmdline.pyindex 68f166849,efc9b36ea..1b579f10e--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@@ -3,11 -3,9 +3,10 @@@ import cProfilimport inspectimport osimport sys-- import pkg_resources+ from importlib.metadata import entry_pointsimport scrapy+from scrapy.addons import AddonManagerfrom scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatterfrom scrapy.crawler import CrawlerProcessfrom scrapy.exceptions import UsageErrordiff --cc scrapy/crawler.pyindex fdcc6354d,69ff07bb7..0c9861dbd--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@@ -199,7 -191,7 +200,11 @@@ class CrawlerRunner)return loader_cls.from_settings(settings.frozencopy())- def __init__(self, settings=None, addons: Optional[AddonManager] = None):- def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):++ def __init__(++ self,++ settings: Union[Dict[str, Any], Settings, None] = None,++ addons: Optional[AddonManager] = None,++ ):if isinstance(settings, dict) or settings is None:settings = Settings(settings)self.settings = settings@@@ -282,10 -275,10 +289,10 @@@return crawler_or_spiderclsreturn self._create_crawler(crawler_or_spidercls)- def _create_crawler(self, spidercls):+ def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:if isinstance(spidercls, str):spidercls = self.spider_loader.load(spidercls)- return Crawler(spidercls, self.settings)+ return Crawler(spidercls, self.settings, addons=self.addons)def stop(self):"""diff --cc scrapy/utils/misc.pyindex 8577cce02,70187ba74..4e23b01c3--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@@ -72,23 -86,7 +86,23 @@@ def load_object(path: Union[str, Callabreturn obj+def load_module_or_object(path):+ """Load python module or (non-module) object from given path.++ Path can be both a Python or a file path.+ """+ try:+ return import_module(path)+ except ImportError:+ pass+ try:+ return load_object(path)+ except (ValueError, NameError, ImportError):+ pass+ raise NameError(f"Could not load '{path}'")++- def walk_modules(path):+ def walk_modules(path: str) -> List[ModuleType]:"""Loads a module and all its submodules from the given module path andreturns them. If *any* module throws an exception while importing, thatexception is thrown back.diff --cc scrapy/utils/test.pyindex 364cca613,97de8d25a..fe26e1708--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@@ -55,10 -66,14 +66,15 @@@ def get_ftp_content_and_deletedirname, filename = split(path)ftp.cwd(dirname)ftp.delete(filename)- return "".join(ftp_data)+ return b"".join(ftp_data)- def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True, addons=None):+ def get_crawler(+ spidercls: Optional[Type[Spider]] = None,+ settings_dict: Optional[Dict[str, Any]] = None,+ prevent_warnings: bool = True,++ addons=None,+ ) -> Crawler:"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project levelpriority.