Scrapy, a fast high-level web crawling & scraping framework for Python.
Add create_instance support to addons.
Andrey Rakhmatullinc92c9af07521
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 3421864fa..f1cc070ad 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project'sWriting your own add-ons========================-Add-ons are (any) Python *objects* that include the following method:+Add-ons are (any) Python objects that include the following method:.. method:: update_settings(settings)@@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method::param settings: The settings object storing Scrapy/component configuration:type settings: :class:`~scrapy.settings.Settings`+They can also have the following method:++.. classmethod:: from_crawler(cls, crawler)+ :noindex:++ If present, this class method is called to create an addon instance+ from a :class:`~scrapy.crawler.Crawler`. It must return a new instance+ of the addon. Crawler object provides access to all Scrapy core+ components like settings and signals; it is a way for pipeline to+ access them and hook its functionality into Scrapy.++ :param crawler: The crawler that uses this addon+ :type crawler: :class:`~scrapy.crawler.Crawler`+Add-on examples===============@@ -67,3 +81,17 @@ Check dependencies::except ImportError:raise RuntimeError("MyAddon requires the boto library")...++Access the crawler instance::++ class MyAddon:+ def __init__(self, crawler) -> None:+ super().__init__()+ self.crawler = crawler++ @classmethod+ def from_crawler(cls, crawler: Crawler):+ return cls(crawler)++ def update_settings(self, settings):+ ...diff --git a/scrapy/addons.py b/scrapy/addons.pyindex bb4664d8e..ba33f1865 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -1,13 +1,17 @@-from typing import Any, List+from typing import TYPE_CHECKING, Any, Listfrom scrapy.utils.conf import build_component_list-from scrapy.utils.misc import load_object+from scrapy.utils.misc import create_instance, load_object++if TYPE_CHECKING:+ from scrapy.crawler import Crawlerclass AddonManager:"""This class facilitates loading and storing :ref:`topics-addons`."""- def __init__(self) -> None:+ def __init__(self, crawler: "Crawler") -> None:+ self.crawler: "Crawler" = crawlerself.addons: List[Any] = []def add(self, addon: Any) -> None:@@ -22,7 +26,7 @@ class AddonManager:if isinstance(addon, (type, str)):addon = load_object(addon)if isinstance(addon, type):- addon = addon()+ addon = create_instance(addon, settings=None, crawler=self.crawler)self.addons.append(addon)def load_settings(self, settings) -> None:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 7a26cd2e4..12256440b 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -69,7 +69,7 @@ class Crawler:self.settings: Settings = settings.copy()self.spidercls.update_settings(self.settings)- self.addons: AddonManager = AddonManager()+ self.addons: AddonManager = AddonManager(self)self.addons.load_settings(self.settings)self.addons.update_settings(self.settings)diff --git a/tests/test_addons.py b/tests/test_addons.pyindex 8ba27236d..d52665869 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -1,13 +1,12 @@import unittestfrom typing import Any, Dict, Optional-from scrapy.addons import AddonManager+from scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings+from scrapy.utils.test import get_crawlerclass GoodAddon:- name = "GoodAddon"-def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:super().__init__()self.config = config or {}@@ -16,6 +15,20 @@ class GoodAddon:settings.update(self.config, "addon")+class CreateInstanceAddon:+ def __init__(self, crawler: Crawler) -> None:+ super().__init__()+ self.crawler = crawler+ self.config = crawler.settings.getdict("MYADDON")++ @classmethod+ def from_crawler(cls, crawler: Crawler):+ return cls(crawler)++ def update_settings(self, settings):+ settings.update(self.config, "addon")++class AddonTest(unittest.TestCase):def test_update_settings(self):settings = BaseSettings()@@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase):class AddonManagerTest(unittest.TestCase):- def setUp(self):- self.manager = AddonManager()-def test_add(self):- manager = AddonManager()+ crawler = get_crawler()+ manager = crawler.addonsmanager.add("tests.test_addons.GoodAddon")self.assertIsInstance(manager.addons[0], GoodAddon)def test_load_settings(self):- settings = BaseSettings()- settings.set(- "ADDONS",- {"tests.test_addons.GoodAddon": 0},- )- settings.set("GOODADDON", {"key": "val2"})- manager = AddonManager()- manager.load_settings(settings)+ settings_dict = {+ "ADDONS": {"tests.test_addons.GoodAddon": 0},+ }+ crawler = get_crawler(settings_dict=settings_dict)+ manager = crawler.addonsself.assertIsInstance(manager.addons[0], GoodAddon)++ def test_create_instance(self):+ settings_dict = {+ "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},+ "MYADDON": {"MYADDON_KEY": "val"},+ }+ crawler = get_crawler(settings_dict=settings_dict)+ manager = crawler.addons+ self.assertIsInstance(manager.addons[0], CreateInstanceAddon)+ self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")