Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove some dead code/docs.

Andrey Rakhmatullin2f9ebb66c32f
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 523f6e86e..35e40ca15 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -17,16 +17,7 @@ Activating and configuring add-ons
Add-ons and their configuration live in Scrapy's
:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and
only then, the add-on manager will read a list of enabled add-ons and their
-configurations from your settings. There are two places where you can provide
-the paths to add-ons you want to enable:
-
-* the ``ADDONS`` setting, and
-* the ``scrapy.cfg`` file.
-
-As Scrapy settings can be modified from many places, e.g. in a project's
-``settings.py``, in a Spider's ``custom_settings`` attribute, or from the
-command line, using the ``ADDONS`` setting is the preferred way to
-manage add-ons.
+configurations from your ``ADDONS`` setting.
The ``ADDONS`` setting a tuple in which every item is a path to an
add-on. The path can be both a Python or a file path. While more precise, it is
@@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's
'some_config': True,
}
-It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is
-a little friendlier, be aware that this file, and therefore the configuration in
-it, is not bound to a particular Scrapy project. While this should not pose a
-problem when you use the project on your development machine only, a common
-stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``.
-
-In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the
-dictionary keys. I.e., the configuration from above would look like this:
-
-.. code-block:: cfg
-
- [addon:httpcache]
- expiration_secs = 60
- ignore_http_codes = 404,405
-
- [addon:path.to.some.addon]
- some_config = true
-
-
Enabling and configuring add-ons within Python code
---------------------------------------------------
The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings
-and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process.
+*at the beginning* of Scrapy's start-up process.
Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is
populated, changing the ``ADDONS`` setting or any of the add-on
configuration dictionary settings will have no effect.
diff --git a/scrapy/addons.py b/scrapy/addons.py
index 43a36e5e5..66463afdb 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -261,32 +261,6 @@ class AddonManager(Mapping):
obj = AddonManager.get_addon(obj._addon)
return obj
- def load_dict(self, addonsdict):
- """Load add-ons and configurations from given dictionary.
-
- Each add-on should be an entry in the dictionary, where the key
- corresponds to the add-on path. The value should be a dictionary
- representing the add-on configuration.
-
- Example add-on dictionary::
-
- addonsdict = {
- 'path.to.addon1': {
- 'setting1': 'value',
- 'setting2': 42,
- },
- 'path/to/addon2.py': {
- 'addon2setting': True,
- },
- }
-
- :param addonsdict: dictionary where keys correspond to add-on paths \
- and values correspond to their configuration
- :type addonsdict: ``dict``
- """
- for addonpath, addoncfg in addonsdict.items():
- self.add(addonpath, addoncfg)
-
def load_settings(self, settings):
"""Load add-ons and configurations from settings object.
diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py
index 451ffed0f..d1c17a044 100644
--- a/tests/test_addons/__init__.py
+++ b/tests/test_addons/__init__.py
@@ -1,7 +1,6 @@
import itertools
import unittest
import warnings
-from collections import OrderedDict
from unittest import mock
from zope.interface import directlyProvides
@@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase):
x._addon._addon = addons.GoodAddon("inner")
self.assertIs(self.manager.get_addon(x), x._addon._addon)
- def test_load_dict_load_settings(self):
- def _test_load_method(func, *args, **kwargs):
- manager = AddonManager()
- getattr(manager, func)(*args, **kwargs)
- self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
- self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
- self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
- self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
- self.assertEqual(manager["AddonModule"], addonmod)
- self.assertIn("key", manager.configs["AddonModule"])
- self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
-
- addonsdict = {
- "tests.test_addons.addonmod": {"key": "val1"},
- "tests.test_addons.addons.GoodAddon": {"key": "val2"},
- }
- _test_load_method("load_dict", addonsdict)
-
+ def test_load_settings(self):
settings = BaseSettings()
settings.set(
"ADDONS",
@@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase):
)
settings.set("ADDONMODULE", {"key": "val1"})
settings.set("GOODADDON", {"key": "val2"})
- _test_load_method("load_settings", settings)
-
- def test_load_dict_load_settings_order(self):
- def _test_load_method(expected_order, func, *args, **kwargs):
- manager = AddonManager()
- getattr(manager, func)(*args, **kwargs)
- self.assertEqual(list(manager.keys()), expected_order)
+ manager = AddonManager()
+ manager.load_settings(settings)
+ self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
+ self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
+ self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
+ self.assertEqual(manager["AddonModule"], addonmod)
+ self.assertIn("key", manager.configs["AddonModule"])
+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
+ def test_load_settings_order(self):
# Get three addons named 0, 1, 2
addonlist = [addons.GoodAddon(str(x)) for x in range(3)]
# Test both methods for every possible mutation
for ordered_addons in itertools.permutations(addonlist):
expected_order = [a.name for a in ordered_addons]
- addonsdict = OrderedDict((a, {}) for a in ordered_addons)
- _test_load_method(expected_order, "load_dict", addonsdict)
settings = BaseSettings(
{"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
)
- _test_load_method(expected_order, "load_settings", settings)
+ manager = AddonManager()
+ manager.load_settings(settings)
+ self.assertEqual(list(manager.keys()), expected_order)
def test_enabled_disabled(self):
manager = AddonManager()