Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle NotConfigured in add-ons.
Andrey Rakhmatullin41a4a163e371
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 8dddb7c91..32c085079 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -77,6 +77,10 @@ modify :setting:`ITEM_PIPELINES`::if settings.getbool("MYADDON_ENABLE_PIPELINE"):settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200+If the ``update_settings`` method raises+:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This+makes it easy to enable an add-on only when some conditions are met.+Fallbacks---------@@ -130,7 +134,7 @@ Check dependencies:try:import botoexcept ImportError:- raise RuntimeError("MyAddon requires the boto library")+ raise NotConfigured("MyAddon requires the boto library")...Access the crawler instance:diff --git a/scrapy/addons.py b/scrapy/addons.pyindex e72c5da98..072143107 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -1,6 +1,8 @@import loggingfrom typing import TYPE_CHECKING, Any, List+from scrapy.exceptions import NotConfigured+from scrapy.settings import Settingsfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.misc import create_instance, load_object@@ -17,19 +19,23 @@ class AddonManager:self.crawler: "Crawler" = crawlerself.addons: List[Any] = []- def _add(self, addon: Any) -> None:+ def _add(self, addon: Any, settings: Settings) -> None:"""Store an add-on."""if isinstance(addon, (type, str)):addon = load_object(addon)if isinstance(addon, type):addon = create_instance(addon, settings=None, crawler=self.crawler)- self.addons.append(addon)+ try:+ addon.update_settings(settings)+ self.addons.append(addon)+ except NotConfigured:+ pass- def load_settings(self, settings) -> None:+ def load_settings(self, settings: Settings) -> None:"""Load add-ons and configurations from a settings object.This will load the add-on for every add-on path in the- ``ADDONS`` setting.+ ``ADDONS`` setting and execute their ``update_settings`` methods.:param settings: The :class:`~scrapy.settings.Settings` object from \which to read the add-on configuration@@ -37,7 +43,7 @@ class AddonManager:"""addons = build_component_list(settings["ADDONS"])for addon in build_component_list(settings["ADDONS"]):- self._add(addon)+ self._add(addon, settings)logger.info("Enabled addons:\n%(addons)s",{@@ -45,13 +51,3 @@ class AddonManager:},extra={"crawler": self.crawler},)-- def update_settings(self, settings) -> None:- """Call ``update_settings()`` of all held add-ons.-- :param settings: The :class:`~scrapy.settings.Settings` object to be \- updated- :type settings: :class:`~scrapy.settings.Settings`- """- for addon in self.addons:- addon.update_settings(settings)diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 12256440b..bf69cee26 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -71,7 +71,6 @@ class Crawler:self.addons: AddonManager = AddonManager(self)self.addons.load_settings(self.settings)- self.addons.update_settings(self.settings)self.signals: SignalManager = SignalManager(self)diff --git a/tests/test_addons.py b/tests/test_addons.pyindex 4156762d3..5d053ed52 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -4,6 +4,7 @@ from typing import Any, Dictfrom scrapy import Spiderfrom scrapy.crawler import Crawler, CrawlerRunner+from scrapy.exceptions import NotConfiguredfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.test import get_crawler@@ -57,6 +58,18 @@ class AddonManagerTest(unittest.TestCase):manager = crawler.addonsself.assertIsInstance(manager.addons[0], SimpleAddon)+ def test_notconfigured(self):+ class NotConfiguredAddon:+ def update_settings(self, settings):+ raise NotConfigured()++ settings_dict = {+ "ADDONS": {NotConfiguredAddon: 0},+ }+ crawler = get_crawler(settings_dict=settings_dict)+ manager = crawler.addons+ self.assertFalse(manager.addons)+def test_load_settings_order(self):# Get three addons with different settingsaddonlist = []