Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add create_instance support to addons.

Andrey Rakhmatullinc92c9af07521
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 3421864fa..f1cc070ad 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's
Writing your own add-ons
========================
-Add-ons are (any) Python *objects* that include the following method:
+Add-ons are (any) Python objects that include the following method:
.. method:: update_settings(settings)
@@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method:
:param settings: The settings object storing Scrapy/component configuration
:type settings: :class:`~scrapy.settings.Settings`
+They can also have the following method:
+
+.. classmethod:: from_crawler(cls, crawler)
+ :noindex:
+
+ If present, this class method is called to create an addon instance
+ from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
+ of the addon. Crawler object provides access to all Scrapy core
+ components like settings and signals; it is a way for pipeline to
+ access them and hook its functionality into Scrapy.
+
+ :param crawler: The crawler that uses this addon
+ :type crawler: :class:`~scrapy.crawler.Crawler`
+
Add-on examples
===============
@@ -67,3 +81,17 @@ Check dependencies::
except ImportError:
raise RuntimeError("MyAddon requires the boto library")
...
+
+Access the crawler instance::
+
+ class MyAddon:
+ def __init__(self, crawler) -> None:
+ super().__init__()
+ self.crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler):
+ return cls(crawler)
+
+ def update_settings(self, settings):
+ ...
diff --git a/scrapy/addons.py b/scrapy/addons.py
index bb4664d8e..ba33f1865 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -1,13 +1,17 @@
-from typing import Any, List
+from typing import TYPE_CHECKING, Any, List
from scrapy.utils.conf import build_component_list
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
+
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
class AddonManager:
"""This class facilitates loading and storing :ref:`topics-addons`."""
- def __init__(self) -> None:
+ def __init__(self, crawler: "Crawler") -> None:
+ self.crawler: "Crawler" = crawler
self.addons: List[Any] = []
def add(self, addon: Any) -> None:
@@ -22,7 +26,7 @@ class AddonManager:
if isinstance(addon, (type, str)):
addon = load_object(addon)
if isinstance(addon, type):
- addon = addon()
+ addon = create_instance(addon, settings=None, crawler=self.crawler)
self.addons.append(addon)
def load_settings(self, settings) -> None:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 7a26cd2e4..12256440b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -69,7 +69,7 @@ class Crawler:
self.settings: Settings = settings.copy()
self.spidercls.update_settings(self.settings)
- self.addons: AddonManager = AddonManager()
+ self.addons: AddonManager = AddonManager(self)
self.addons.load_settings(self.settings)
self.addons.update_settings(self.settings)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 8ba27236d..d52665869 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -1,13 +1,12 @@
import unittest
from typing import Any, Dict, Optional
-from scrapy.addons import AddonManager
+from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
+from scrapy.utils.test import get_crawler
class GoodAddon:
- name = "GoodAddon"
-
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
super().__init__()
self.config = config or {}
@@ -16,6 +15,20 @@ class GoodAddon:
settings.update(self.config, "addon")
+class CreateInstanceAddon:
+ def __init__(self, crawler: Crawler) -> None:
+ super().__init__()
+ self.crawler = crawler
+ self.config = crawler.settings.getdict("MYADDON")
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler):
+ return cls(crawler)
+
+ def update_settings(self, settings):
+ settings.update(self.config, "addon")
+
+
class AddonTest(unittest.TestCase):
def test_update_settings(self):
settings = BaseSettings()
@@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase):
class AddonManagerTest(unittest.TestCase):
- def setUp(self):
- self.manager = AddonManager()
-
def test_add(self):
- manager = AddonManager()
+ crawler = get_crawler()
+ manager = crawler.addons
manager.add("tests.test_addons.GoodAddon")
self.assertIsInstance(manager.addons[0], GoodAddon)
def test_load_settings(self):
- settings = BaseSettings()
- settings.set(
- "ADDONS",
- {"tests.test_addons.GoodAddon": 0},
- )
- settings.set("GOODADDON", {"key": "val2"})
- manager = AddonManager()
- manager.load_settings(settings)
+ settings_dict = {
+ "ADDONS": {"tests.test_addons.GoodAddon": 0},
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ manager = crawler.addons
self.assertIsInstance(manager.addons[0], GoodAddon)
+
+ def test_create_instance(self):
+ settings_dict = {
+ "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},
+ "MYADDON": {"MYADDON_KEY": "val"},
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ manager = crawler.addons
+ self.assertIsInstance(manager.addons[0], CreateInstanceAddon)
+ self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")