Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove some dead code/docs.
Andrey Rakhmatullin2f9ebb66c32f
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 523f6e86e..35e40ca15 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -17,16 +17,7 @@ Activating and configuring add-onsAdd-ons and their configuration live in Scrapy's:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, andonly then, the add-on manager will read a list of enabled add-ons and their-configurations from your settings. There are two places where you can provide-the paths to add-ons you want to enable:--* the ``ADDONS`` setting, and-* the ``scrapy.cfg`` file.--As Scrapy settings can be modified from many places, e.g. in a project's-``settings.py``, in a Spider's ``custom_settings`` attribute, or from the-command line, using the ``ADDONS`` setting is the preferred way to-manage add-ons.+configurations from your ``ADDONS`` setting.The ``ADDONS`` setting a tuple in which every item is a path to anadd-on. The path can be both a Python or a file path. While more precise, it is@@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's'some_config': True,}-It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is-a little friendlier, be aware that this file, and therefore the configuration in-it, is not bound to a particular Scrapy project. While this should not pose a-problem when you use the project on your development machine only, a common-stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``.--In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the-dictionary keys. I.e., the configuration from above would look like this:--.. code-block:: cfg-- [addon:httpcache]- expiration_secs = 60- ignore_http_codes = 404,405-- [addon:path.to.some.addon]- some_config = true--Enabling and configuring add-ons within Python code---------------------------------------------------The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings-and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process.+*at the beginning* of Scrapy's start-up process.Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` ispopulated, changing the ``ADDONS`` setting or any of the add-onconfiguration dictionary settings will have no effect.diff --git a/scrapy/addons.py b/scrapy/addons.pyindex 43a36e5e5..66463afdb 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -261,32 +261,6 @@ class AddonManager(Mapping):obj = AddonManager.get_addon(obj._addon)return obj- def load_dict(self, addonsdict):- """Load add-ons and configurations from given dictionary.-- Each add-on should be an entry in the dictionary, where the key- corresponds to the add-on path. The value should be a dictionary- representing the add-on configuration.-- Example add-on dictionary::-- addonsdict = {- 'path.to.addon1': {- 'setting1': 'value',- 'setting2': 42,- },- 'path/to/addon2.py': {- 'addon2setting': True,- },- }-- :param addonsdict: dictionary where keys correspond to add-on paths \- and values correspond to their configuration- :type addonsdict: ``dict``- """- for addonpath, addoncfg in addonsdict.items():- self.add(addonpath, addoncfg)-def load_settings(self, settings):"""Load add-ons and configurations from settings object.diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.pyindex 451ffed0f..d1c17a044 100644--- a/tests/test_addons/__init__.py+++ b/tests/test_addons/__init__.py@@ -1,7 +1,6 @@import itertoolsimport unittestimport warnings-from collections import OrderedDictfrom unittest import mockfrom zope.interface import directlyProvides@@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase):x._addon._addon = addons.GoodAddon("inner")self.assertIs(self.manager.get_addon(x), x._addon._addon)- def test_load_dict_load_settings(self):- def _test_load_method(func, *args, **kwargs):- manager = AddonManager()- getattr(manager, func)(*args, **kwargs)- self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])- self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)- self.assertCountEqual(manager.configs["GoodAddon"], ["key"])- self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")- self.assertEqual(manager["AddonModule"], addonmod)- self.assertIn("key", manager.configs["AddonModule"])- self.assertEqual(manager.configs["AddonModule"]["key"], "val1")-- addonsdict = {- "tests.test_addons.addonmod": {"key": "val1"},- "tests.test_addons.addons.GoodAddon": {"key": "val2"},- }- _test_load_method("load_dict", addonsdict)-+ def test_load_settings(self):settings = BaseSettings()settings.set("ADDONS",@@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase):)settings.set("ADDONMODULE", {"key": "val1"})settings.set("GOODADDON", {"key": "val2"})- _test_load_method("load_settings", settings)-- def test_load_dict_load_settings_order(self):- def _test_load_method(expected_order, func, *args, **kwargs):- manager = AddonManager()- getattr(manager, func)(*args, **kwargs)- self.assertEqual(list(manager.keys()), expected_order)+ manager = AddonManager()+ manager.load_settings(settings)+ self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)+ self.assertCountEqual(manager.configs["GoodAddon"], ["key"])+ self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")+ self.assertEqual(manager["AddonModule"], addonmod)+ self.assertIn("key", manager.configs["AddonModule"])+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")+ def test_load_settings_order(self):# Get three addons named 0, 1, 2addonlist = [addons.GoodAddon(str(x)) for x in range(3)]# Test both methods for every possible mutationfor ordered_addons in itertools.permutations(addonlist):expected_order = [a.name for a in ordered_addons]- addonsdict = OrderedDict((a, {}) for a in ordered_addons)- _test_load_method(expected_order, "load_dict", addonsdict)settings = BaseSettings({"ADDONS": {a: i for i, a in enumerate(ordered_addons)}})- _test_load_method(expected_order, "load_settings", settings)+ manager = AddonManager()+ manager.load_settings(settings)+ self.assertEqual(list(manager.keys()), expected_order)def test_enabled_disabled(self):manager = AddonManager()