Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove more code.
Andrey Rakhmatullin9612ae3e9323
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 6a9590f33..3421864fa 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -17,40 +17,26 @@ Activating and configuring add-onsAdd-ons and their configuration live in Scrapy's:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler`initialization the add-on manager will read a list of enabled add-ons from your-``ADDONS`` setting and their optional configuration from the respective-settings.+``ADDONS`` setting.The ``ADDONS`` setting is a dict in which every key is an addon class or itsimport path and the value is its priority.-The configuration of an add-on, if necessary at all, is stored as a dictionary-setting whose name is the uppercase add-on name.--This is an example where two add-ons (in this case with one requiring no-configuration) are enabled/configured in a project's ``settings.py``::+This is an example where two add-ons are enabled in a project's+``settings.py``::ADDONS = {'path.to.someaddon': 0,path.to.someaddon2: 1,}- SOMEADDON = {- 'some_config': True,- }-Writing your own add-ons========================-Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:--.. attribute:: name+Add-ons are (any) Python *objects* that include the following method:- string with add-on name-- :type: ``str``--.. method:: update_settings(config, settings)+.. method:: update_settings(settings)This method is called during the initialization of the:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks@@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*::class:`~scrapy.settings.Settings` object as wished, e.g. enable componentsfor this add-on or set required configuration of other extensions.- :param config: Configuration of this add-on- :type config: ``dict``-:param settings: The settings object storing Scrapy/component configuration:type settings: :class:`~scrapy.settings.Settings`-.. method:: check_configuration(config, crawler)-- This method is called when the :class:`~scrapy.crawler.Crawler` has been- fully initialized, immediately before it starts crawling. You can perform- additional dependency and configuration checks here.-- :param config: Configuration of this add-on- :type config: ``dict``-- :param crawler: Fully initialized Scrapy crawler- :type crawler: :class:`~scrapy.crawler.Crawler`---Add-on base class-=================--Scrapy comes with a built-in base class for add-ons which provides some-convenience functionality: the add-on configuration can be exposed into-Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable-via :attr:`~scrapy.addons.Addon.default_config` and-:attr:`~scrapy.addons.Addon.config_mapping`.--By default, the base add-on class will expose the add-on configuration into-Scrapy's settings namespace, in upper case. It is-easy to write your own functionality while still being able to use the-convenience functions by overwriting-:meth:`~scrapy.addons.Addon.update_settings`.--.. module:: scrapy.addons- :noindex:--.. autoclass:: Addon- :members:-Add-on examples===============-Set some basic configuration using the :class:`Addon` base class::-- from scrapy.addons import Addon-- class MyAddon(Addon):- name = 'myaddon'+Set some basic configuration::- def update_settings(self, config, settings):- super().update_settings(settings)+ class MyAddon:+ def update_settings(self, settings):settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200settings["DNSCACHE_ENABLED"] = TrueCheck dependencies::- from scrapy.addons import Addon-- class MyAddon(Addon):- name = 'myaddon'-- def update_settings(self, config, settings):+ class MyAddon:+ def update_settings(self, settings):try:import botoexcept ImportError:- raise RuntimeError("myaddon requires the boto library")- super().update_settings(settings)--Check configuration of fully initialized crawler (see-:ref:`topics-api-crawler`)::-- class MyAddon(object):- name = 'myaddon'-- def update_settings(self, config, settings):- super().update_settings(settings)- settings.set('DNSCACHE_ENABLED', False, priority='addon')-- def check_configuration(self, config, crawler):- if crawler.settings.getbool('DNSCACHE_ENABLED'):- # The spider, some other add-on, or the user messed with the- # DNS cache setting- raise ValueError("myaddon is incompatible with DNS cache")+ raise RuntimeError("MyAddon requires the boto library")+ ...diff --git a/scrapy/addons.py b/scrapy/addons.pyindex 523c36e9c..bb4664d8e 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -1,100 +1,16 @@-from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict+from typing import Any, Listfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.misc import load_object-class Addon(object):- name: str-- default_config = None- """``dict`` with default configuration."""-- config_mapping = None- """``dict`` with mappings from config names to setting names. The given- setting names will be taken as given, not uppercased.- """-- def export_config(self, config, settings):- """Export the add-on configuration, all keys in caps, into the settings- object.-- For example, the add-on configuration ``{'key': 'value'}`` will export- the setting ``KEY`` with a value of ``value``. All settings- will be exported with ``addon`` priority (see- :ref:`topics-api-settings`).-- :param config: Add-on configuration to be exposed- :type config: ``dict``-- :param settings: Settings object into which to export the configuration- :type settings: :class:`~scrapy.settings.Settings`- """- conf = self.default_config or {}- conf.update(config)- # Since default exported config is case-insensitive (everything will be- # uppercased), make mapped config case-insensitive as well- conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}- for key, val in conf.items():- if key.lower() in conf_mapping:- key = conf_mapping[key.lower()]- else:- key = key.upper()- settings.set(key, val, "addon")-- def update_settings(self, config, settings):- """Modifiy `settings` to enable and configure required components.-- :param config: Add-on configuration- :type config: ``dict``-- :param settings: Crawler settings object- :type settings: :class:`~scrapy.settings.Settings`- """- self.export_config(config, settings)-- def check_configuration(self, config, crawler):- """Perform post-initialization checks on fully configured `crawler`.-- :param config: Add-on configuration- :type config: ``dict``-- :param crawler: the fully-initialized crawler- :type crawler: :class:`~scrapy.crawler.Crawler`- """- pass---class AddonManager(Mapping[str, Addon]):- """This class facilitates loading and storing :ref:`topics-addons`.-- You can treat it like a read-only dictionary in which keys correspond to- add-on names and values correspond to the add-on objects. Add-on- configurations are saved in the :attr:`config` dictionary attribute::-- addons = AddonManager()- # ... load some add-ons here- print(addons.enabled) # prints names of all enabled add-ons- print(addons['TestAddon'].version) # prints version of add-on with name- # 'TestAddon'- print(addons.configs['TestAddon']) # prints configuration of 'TestAddon'-- """+class AddonManager:+ """This class facilitates loading and storing :ref:`topics-addons`."""def __init__(self) -> None:- self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]()- self.configs: Dict[str, Dict[str, Any]] = {}-- def __getitem__(self, name: str) -> Addon:- return self._addons[name]-- def __iter__(self) -> Iterator[str]:- return iter(self._addons)-- def __len__(self) -> int:- return len(self._addons)+ self.addons: List[Any] = []- def add(self, addon: Any, config: Optional[Dict[str, Any]] = None):+ def add(self, addon: Any) -> None:"""Store an add-on.:param addon: The add-on object (or path) to be stored@@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]):addon = load_object(addon)if isinstance(addon, type):addon = addon()- name = addon.name- if name in self:- raise ValueError(f"Addon '{name}' already loaded")- self._addons[name] = addon- self.configs[name] = config or {}+ self.addons.append(addon)- def load_settings(self, settings):+ def load_settings(self, settings) -> None:"""Load add-ons and configurations from settings object.This will load the addon for every add-on path in the- ``ADDONS`` setting. For each of these add-ons, the configuration will be- read from the dictionary setting whose name matches the uppercase add-on- name.+ ``ADDONS`` setting.:param settings: The :class:`~scrapy.settings.Settings` object from \which to read the add-on configuration@@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]):"""paths = build_component_list(settings["ADDONS"])addons = [load_object(path) for path in paths]- configs = [settings.getdict(addon.name.upper()) for addon in addons]- for a, c in zip(addons, configs):- self.add(a, c)+ for a in addons:+ self.add(a)def update_settings(self, settings) -> None:"""Call ``update_settings()`` of all held add-ons.@@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]):updated:type settings: :class:`~scrapy.settings.Settings`"""- for name in self:- self[name].update_settings(self.configs[name], settings)-- def check_configuration(self, crawler) -> None:- """Call ``check_configuration()`` of all held add-ons.-- :param crawler: the fully-initialized crawler- :type crawler: :class:`~scrapy.crawler.Crawler`- """- for name in self:- self[name].check_configuration(self.configs[name], crawler)+ for addon in self.addons:+ addon.update_settings(settings)diff --git a/tests/test_addons.py b/tests/test_addons.pyindex ecdc0426c..8ba27236d 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -1,47 +1,29 @@import unittest+from typing import Any, Dict, Optional-from scrapy.addons import Addon, AddonManager+from scrapy.addons import AddonManagerfrom scrapy.settings import BaseSettings-class GoodAddon(object):+class GoodAddon:name = "GoodAddon"- def update_settings(self, config, settings):- pass+ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:+ super().__init__()+ self.config = config or {}- def check_configuration(self, config, crawler):- pass+ def update_settings(self, settings):+ settings.update(self.config, "addon")class AddonTest(unittest.TestCase):- def setUp(self):- class AddonWithAttributes(Addon):- name = "Test"-- self.testaddon = AddonWithAttributes()-- def test_export_config(self):- settings = BaseSettings()- self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}- self.testaddon.default_config = {"key": 55, "defaultkey": 100}- self.testaddon.export_config(- {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings- )- self.assertEqual(settings["KEY"], 313)- self.assertEqual(settings["DEFAULTKEY"], 100)- self.assertEqual(settings["OTHERKEY"], True)- self.assertNotIn("MAPPED_key", settings)- self.assertNotIn("MAPPED_KEY", settings)- self.assertEqual(settings["MAPPING_WORKED"], 99)- self.assertEqual(settings.getpriority("KEY"), 15)-def test_update_settings(self):settings = BaseSettings()settings.set("KEY1", "default", priority="default")settings.set("KEY2", "project", priority="project")- addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}- self.testaddon.update_settings(addon_config, settings)+ addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"}+ testaddon = GoodAddon(addon_config)+ testaddon.update_settings(settings)self.assertEqual(settings["KEY1"], "addon")self.assertEqual(settings["KEY2"], "project")self.assertEqual(settings["KEY3"], "addon")@@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase):def test_add(self):manager = AddonManager()manager.add("tests.test_addons.GoodAddon")- self.assertCountEqual(manager, ["GoodAddon"])- self.assertIsInstance(manager["GoodAddon"], GoodAddon)+ self.assertIsInstance(manager.addons[0], GoodAddon)def test_load_settings(self):settings = BaseSettings()@@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase):settings.set("GOODADDON", {"key": "val2"})manager = AddonManager()manager.load_settings(settings)- self.assertCountEqual(manager, ["GoodAddon"])- self.assertIsInstance(manager["GoodAddon"], GoodAddon)- self.assertCountEqual(manager.configs["GoodAddon"], ["key"])- self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")+ self.assertIsInstance(manager.addons[0], GoodAddon)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex d844a645f..ca9084294 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -13,7 +13,6 @@ from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom scrapy import signals-from scrapy.addons import Addon, AddonManagerfrom scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import StopDownloadfrom scrapy.http import Request@@ -408,21 +407,6 @@ with multiples linesself._assert_retried(log)self.assertIn("Got response 200", str(log))- @defer.inlineCallbacks- def test_abort_on_addon_failed_check(self):- class FailedCheckAddon(Addon):- name = "FailedCheckAddon"-- def check_configuration(self, config, crawler):- raise ValueError-- addonmgr = AddonManager()- addonmgr.add(FailedCheckAddon())- crawler = get_crawler(SimpleSpider)- crawler.addons = addonmgr- with self.assertRaises(ValueError):- yield crawler.crawl()-class CrawlSpiderTestCase(TestCase):def setUp(self):