Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove more code.

Andrey Rakhmatullin9612ae3e9323
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 6a9590f33..3421864fa 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -17,40 +17,26 @@ Activating and configuring add-ons
Add-ons and their configuration live in Scrapy's
:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler`
initialization the add-on manager will read a list of enabled add-ons from your
-``ADDONS`` setting and their optional configuration from the respective
-settings.
+``ADDONS`` setting.
The ``ADDONS`` setting is a dict in which every key is an addon class or its
import path and the value is its priority.
-The configuration of an add-on, if necessary at all, is stored as a dictionary
-setting whose name is the uppercase add-on name.
-
-This is an example where two add-ons (in this case with one requiring no
-configuration) are enabled/configured in a project's ``settings.py``::
+This is an example where two add-ons are enabled in a project's
+``settings.py``::
ADDONS = {
'path.to.someaddon': 0,
path.to.someaddon2: 1,
}
- SOMEADDON = {
- 'some_config': True,
- }
-
Writing your own add-ons
========================
-Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
-
-.. attribute:: name
+Add-ons are (any) Python *objects* that include the following method:
- string with add-on name
-
- :type: ``str``
-
-.. method:: update_settings(config, settings)
+.. method:: update_settings(settings)
This method is called during the initialization of the
:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
@@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
:class:`~scrapy.settings.Settings` object as wished, e.g. enable components
for this add-on or set required configuration of other extensions.
- :param config: Configuration of this add-on
- :type config: ``dict``
-
:param settings: The settings object storing Scrapy/component configuration
:type settings: :class:`~scrapy.settings.Settings`
-.. method:: check_configuration(config, crawler)
-
- This method is called when the :class:`~scrapy.crawler.Crawler` has been
- fully initialized, immediately before it starts crawling. You can perform
- additional dependency and configuration checks here.
-
- :param config: Configuration of this add-on
- :type config: ``dict``
-
- :param crawler: Fully initialized Scrapy crawler
- :type crawler: :class:`~scrapy.crawler.Crawler`
-
-
-Add-on base class
-=================
-
-Scrapy comes with a built-in base class for add-ons which provides some
-convenience functionality: the add-on configuration can be exposed into
-Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable
-via :attr:`~scrapy.addons.Addon.default_config` and
-:attr:`~scrapy.addons.Addon.config_mapping`.
-
-By default, the base add-on class will expose the add-on configuration into
-Scrapy's settings namespace, in upper case. It is
-easy to write your own functionality while still being able to use the
-convenience functions by overwriting
-:meth:`~scrapy.addons.Addon.update_settings`.
-
-.. module:: scrapy.addons
- :noindex:
-
-.. autoclass:: Addon
- :members:
-
Add-on examples
===============
-Set some basic configuration using the :class:`Addon` base class::
-
- from scrapy.addons import Addon
-
- class MyAddon(Addon):
- name = 'myaddon'
+Set some basic configuration::
- def update_settings(self, config, settings):
- super().update_settings(settings)
+ class MyAddon:
+ def update_settings(self, settings):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
settings["DNSCACHE_ENABLED"] = True
Check dependencies::
- from scrapy.addons import Addon
-
- class MyAddon(Addon):
- name = 'myaddon'
-
- def update_settings(self, config, settings):
+ class MyAddon:
+ def update_settings(self, settings):
try:
import boto
except ImportError:
- raise RuntimeError("myaddon requires the boto library")
- super().update_settings(settings)
-
-Check configuration of fully initialized crawler (see
-:ref:`topics-api-crawler`)::
-
- class MyAddon(object):
- name = 'myaddon'
-
- def update_settings(self, config, settings):
- super().update_settings(settings)
- settings.set('DNSCACHE_ENABLED', False, priority='addon')
-
- def check_configuration(self, config, crawler):
- if crawler.settings.getbool('DNSCACHE_ENABLED'):
- # The spider, some other add-on, or the user messed with the
- # DNS cache setting
- raise ValueError("myaddon is incompatible with DNS cache")
+ raise RuntimeError("MyAddon requires the boto library")
+ ...
diff --git a/scrapy/addons.py b/scrapy/addons.py
index 523c36e9c..bb4664d8e 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -1,100 +1,16 @@
-from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict
+from typing import Any, List
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import load_object
-class Addon(object):
- name: str
-
- default_config = None
- """``dict`` with default configuration."""
-
- config_mapping = None
- """``dict`` with mappings from config names to setting names. The given
- setting names will be taken as given, not uppercased.
- """
-
- def export_config(self, config, settings):
- """Export the add-on configuration, all keys in caps, into the settings
- object.
-
- For example, the add-on configuration ``{'key': 'value'}`` will export
- the setting ``KEY`` with a value of ``value``. All settings
- will be exported with ``addon`` priority (see
- :ref:`topics-api-settings`).
-
- :param config: Add-on configuration to be exposed
- :type config: ``dict``
-
- :param settings: Settings object into which to export the configuration
- :type settings: :class:`~scrapy.settings.Settings`
- """
- conf = self.default_config or {}
- conf.update(config)
- # Since default exported config is case-insensitive (everything will be
- # uppercased), make mapped config case-insensitive as well
- conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}
- for key, val in conf.items():
- if key.lower() in conf_mapping:
- key = conf_mapping[key.lower()]
- else:
- key = key.upper()
- settings.set(key, val, "addon")
-
- def update_settings(self, config, settings):
- """Modifiy `settings` to enable and configure required components.
-
- :param config: Add-on configuration
- :type config: ``dict``
-
- :param settings: Crawler settings object
- :type settings: :class:`~scrapy.settings.Settings`
- """
- self.export_config(config, settings)
-
- def check_configuration(self, config, crawler):
- """Perform post-initialization checks on fully configured `crawler`.
-
- :param config: Add-on configuration
- :type config: ``dict``
-
- :param crawler: the fully-initialized crawler
- :type crawler: :class:`~scrapy.crawler.Crawler`
- """
- pass
-
-
-class AddonManager(Mapping[str, Addon]):
- """This class facilitates loading and storing :ref:`topics-addons`.
-
- You can treat it like a read-only dictionary in which keys correspond to
- add-on names and values correspond to the add-on objects. Add-on
- configurations are saved in the :attr:`config` dictionary attribute::
-
- addons = AddonManager()
- # ... load some add-ons here
- print(addons.enabled) # prints names of all enabled add-ons
- print(addons['TestAddon'].version) # prints version of add-on with name
- # 'TestAddon'
- print(addons.configs['TestAddon']) # prints configuration of 'TestAddon'
-
- """
+class AddonManager:
+ """This class facilitates loading and storing :ref:`topics-addons`."""
def __init__(self) -> None:
- self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]()
- self.configs: Dict[str, Dict[str, Any]] = {}
-
- def __getitem__(self, name: str) -> Addon:
- return self._addons[name]
-
- def __iter__(self) -> Iterator[str]:
- return iter(self._addons)
-
- def __len__(self) -> int:
- return len(self._addons)
+ self.addons: List[Any] = []
- def add(self, addon: Any, config: Optional[Dict[str, Any]] = None):
+ def add(self, addon: Any) -> None:
"""Store an add-on.
:param addon: The add-on object (or path) to be stored
@@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]):
addon = load_object(addon)
if isinstance(addon, type):
addon = addon()
- name = addon.name
- if name in self:
- raise ValueError(f"Addon '{name}' already loaded")
- self._addons[name] = addon
- self.configs[name] = config or {}
+ self.addons.append(addon)
- def load_settings(self, settings):
+ def load_settings(self, settings) -> None:
"""Load add-ons and configurations from settings object.
This will load the addon for every add-on path in the
- ``ADDONS`` setting. For each of these add-ons, the configuration will be
- read from the dictionary setting whose name matches the uppercase add-on
- name.
+ ``ADDONS`` setting.
:param settings: The :class:`~scrapy.settings.Settings` object from \
which to read the add-on configuration
@@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]):
"""
paths = build_component_list(settings["ADDONS"])
addons = [load_object(path) for path in paths]
- configs = [settings.getdict(addon.name.upper()) for addon in addons]
- for a, c in zip(addons, configs):
- self.add(a, c)
+ for a in addons:
+ self.add(a)
def update_settings(self, settings) -> None:
"""Call ``update_settings()`` of all held add-ons.
@@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]):
updated
:type settings: :class:`~scrapy.settings.Settings`
"""
- for name in self:
- self[name].update_settings(self.configs[name], settings)
-
- def check_configuration(self, crawler) -> None:
- """Call ``check_configuration()`` of all held add-ons.
-
- :param crawler: the fully-initialized crawler
- :type crawler: :class:`~scrapy.crawler.Crawler`
- """
- for name in self:
- self[name].check_configuration(self.configs[name], crawler)
+ for addon in self.addons:
+ addon.update_settings(settings)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index ecdc0426c..8ba27236d 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -1,47 +1,29 @@
import unittest
+from typing import Any, Dict, Optional
-from scrapy.addons import Addon, AddonManager
+from scrapy.addons import AddonManager
from scrapy.settings import BaseSettings
-class GoodAddon(object):
+class GoodAddon:
name = "GoodAddon"
- def update_settings(self, config, settings):
- pass
+ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
+ super().__init__()
+ self.config = config or {}
- def check_configuration(self, config, crawler):
- pass
+ def update_settings(self, settings):
+ settings.update(self.config, "addon")
class AddonTest(unittest.TestCase):
- def setUp(self):
- class AddonWithAttributes(Addon):
- name = "Test"
-
- self.testaddon = AddonWithAttributes()
-
- def test_export_config(self):
- settings = BaseSettings()
- self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}
- self.testaddon.default_config = {"key": 55, "defaultkey": 100}
- self.testaddon.export_config(
- {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings
- )
- self.assertEqual(settings["KEY"], 313)
- self.assertEqual(settings["DEFAULTKEY"], 100)
- self.assertEqual(settings["OTHERKEY"], True)
- self.assertNotIn("MAPPED_key", settings)
- self.assertNotIn("MAPPED_KEY", settings)
- self.assertEqual(settings["MAPPING_WORKED"], 99)
- self.assertEqual(settings.getpriority("KEY"), 15)
-
def test_update_settings(self):
settings = BaseSettings()
settings.set("KEY1", "default", priority="default")
settings.set("KEY2", "project", priority="project")
- addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}
- self.testaddon.update_settings(addon_config, settings)
+ addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"}
+ testaddon = GoodAddon(addon_config)
+ testaddon.update_settings(settings)
self.assertEqual(settings["KEY1"], "addon")
self.assertEqual(settings["KEY2"], "project")
self.assertEqual(settings["KEY3"], "addon")
@@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase):
def test_add(self):
manager = AddonManager()
manager.add("tests.test_addons.GoodAddon")
- self.assertCountEqual(manager, ["GoodAddon"])
- self.assertIsInstance(manager["GoodAddon"], GoodAddon)
+ self.assertIsInstance(manager.addons[0], GoodAddon)
def test_load_settings(self):
settings = BaseSettings()
@@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase):
settings.set("GOODADDON", {"key": "val2"})
manager = AddonManager()
manager.load_settings(settings)
- self.assertCountEqual(manager, ["GoodAddon"])
- self.assertIsInstance(manager["GoodAddon"], GoodAddon)
- self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
- self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
+ self.assertIsInstance(manager.addons[0], GoodAddon)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index d844a645f..ca9084294 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -13,7 +13,6 @@ from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy import signals
-from scrapy.addons import Addon, AddonManager
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import StopDownload
from scrapy.http import Request
@@ -408,21 +407,6 @@ with multiples lines
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
- @defer.inlineCallbacks
- def test_abort_on_addon_failed_check(self):
- class FailedCheckAddon(Addon):
- name = "FailedCheckAddon"
-
- def check_configuration(self, config, crawler):
- raise ValueError
-
- addonmgr = AddonManager()
- addonmgr.add(FailedCheckAddon())
- crawler = get_crawler(SimpleSpider)
- crawler.addons = addonmgr
- with self.assertRaises(ValueError):
- yield crawler.crawl()
-
class CrawlSpiderTestCase(TestCase):
def setUp(self):