Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into addons

Andrey Rakhmatullin4e21e3e59df6
diff --cc scrapy/cmdline.py
index 68f166849,efc9b36ea..1b579f10e
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@@ -3,11 -3,9 +3,10 @@@ import cProfil
import inspect
import os
import sys
-
- import pkg_resources
+ from importlib.metadata import entry_points
import scrapy
+from scrapy.addons import AddonManager
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
diff --cc scrapy/crawler.py
index fdcc6354d,69ff07bb7..0c9861dbd
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@@ -199,7 -191,7 +200,11 @@@ class CrawlerRunner
)
return loader_cls.from_settings(settings.frozencopy())
- def __init__(self, settings=None, addons: Optional[AddonManager] = None):
- def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):
++ def __init__(
++ self,
++ settings: Union[Dict[str, Any], Settings, None] = None,
++ addons: Optional[AddonManager] = None,
++ ):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
@@@ -282,10 -275,10 +289,10 @@@
return crawler_or_spidercls
return self._create_crawler(crawler_or_spidercls)
- def _create_crawler(self, spidercls):
+ def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
- return Crawler(spidercls, self.settings)
+ return Crawler(spidercls, self.settings, addons=self.addons)
def stop(self):
"""
diff --cc scrapy/utils/misc.py
index 8577cce02,70187ba74..4e23b01c3
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@@ -72,23 -86,7 +86,23 @@@ def load_object(path: Union[str, Callab
return obj
+def load_module_or_object(path):
+ """Load python module or (non-module) object from given path.
+
+ Path can be both a Python or a file path.
+ """
+ try:
+ return import_module(path)
+ except ImportError:
+ pass
+ try:
+ return load_object(path)
+ except (ValueError, NameError, ImportError):
+ pass
+ raise NameError(f"Could not load '{path}'")
+
+
- def walk_modules(path):
+ def walk_modules(path: str) -> List[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that
exception is thrown back.
diff --cc scrapy/utils/test.py
index 364cca613,97de8d25a..fe26e1708
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@@ -55,10 -66,14 +66,15 @@@ def get_ftp_content_and_delete
dirname, filename = split(path)
ftp.cwd(dirname)
ftp.delete(filename)
- return "".join(ftp_data)
+ return b"".join(ftp_data)
- def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True, addons=None):
+ def get_crawler(
+ spidercls: Optional[Type[Spider]] = None,
+ settings_dict: Optional[Dict[str, Any]] = None,
+ prevent_warnings: bool = True,
++ addons=None,
+ ) -> Crawler:
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
priority.