Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle NotConfigured in add-ons.

Andrey Rakhmatullin41a4a163e371
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 8dddb7c91..32c085079 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -77,6 +77,10 @@ modify :setting:`ITEM_PIPELINES`::
if settings.getbool("MYADDON_ENABLE_PIPELINE"):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
+If the ``update_settings`` method raises
+:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This
+makes it easy to enable an add-on only when some conditions are met.
+
Fallbacks
---------
@@ -130,7 +134,7 @@ Check dependencies:
try:
import boto
except ImportError:
- raise RuntimeError("MyAddon requires the boto library")
+ raise NotConfigured("MyAddon requires the boto library")
...
Access the crawler instance:
diff --git a/scrapy/addons.py b/scrapy/addons.py
index e72c5da98..072143107 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -1,6 +1,8 @@
import logging
from typing import TYPE_CHECKING, Any, List
+from scrapy.exceptions import NotConfigured
+from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import create_instance, load_object
@@ -17,19 +19,23 @@ class AddonManager:
self.crawler: "Crawler" = crawler
self.addons: List[Any] = []
- def _add(self, addon: Any) -> None:
+ def _add(self, addon: Any, settings: Settings) -> None:
"""Store an add-on."""
if isinstance(addon, (type, str)):
addon = load_object(addon)
if isinstance(addon, type):
addon = create_instance(addon, settings=None, crawler=self.crawler)
- self.addons.append(addon)
+ try:
+ addon.update_settings(settings)
+ self.addons.append(addon)
+ except NotConfigured:
+ pass
- def load_settings(self, settings) -> None:
+ def load_settings(self, settings: Settings) -> None:
"""Load add-ons and configurations from a settings object.
This will load the add-on for every add-on path in the
- ``ADDONS`` setting.
+ ``ADDONS`` setting and execute their ``update_settings`` methods.
:param settings: The :class:`~scrapy.settings.Settings` object from \
which to read the add-on configuration
@@ -37,7 +43,7 @@ class AddonManager:
"""
addons = build_component_list(settings["ADDONS"])
for addon in build_component_list(settings["ADDONS"]):
- self._add(addon)
+ self._add(addon, settings)
logger.info(
"Enabled addons:\n%(addons)s",
{
@@ -45,13 +51,3 @@ class AddonManager:
},
extra={"crawler": self.crawler},
)
-
- def update_settings(self, settings) -> None:
- """Call ``update_settings()`` of all held add-ons.
-
- :param settings: The :class:`~scrapy.settings.Settings` object to be \
- updated
- :type settings: :class:`~scrapy.settings.Settings`
- """
- for addon in self.addons:
- addon.update_settings(settings)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 12256440b..bf69cee26 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -71,7 +71,6 @@ class Crawler:
self.addons: AddonManager = AddonManager(self)
self.addons.load_settings(self.settings)
- self.addons.update_settings(self.settings)
self.signals: SignalManager = SignalManager(self)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 4156762d3..5d053ed52 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -4,6 +4,7 @@ from typing import Any, Dict
from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerRunner
+from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.test import get_crawler
@@ -57,6 +58,18 @@ class AddonManagerTest(unittest.TestCase):
manager = crawler.addons
self.assertIsInstance(manager.addons[0], SimpleAddon)
+ def test_notconfigured(self):
+ class NotConfiguredAddon:
+ def update_settings(self, settings):
+ raise NotConfigured()
+
+ settings_dict = {
+ "ADDONS": {NotConfiguredAddon: 0},
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ manager = crawler.addons
+ self.assertFalse(manager.addons)
+
def test_load_settings_order(self):
# Get three addons with different settings
addonlist = []