Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove builtin addons.

Andrey Rakhmatullin55ac26228b85
diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py
similarity index 99%
rename from scrapy/addons/__init__.py
rename to scrapy/addons.py
index aad9ef789..c9b873039 100644
--- a/scrapy/addons/__init__.py
+++ b/scrapy/addons.py
@@ -456,6 +456,3 @@ class AddonManager(Mapping):
"""
for name in self:
self._call_addon(name, "check_configuration", crawler)
-
-
-from scrapy.addons.builtins import * # noqa
diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py
deleted file mode 100644
index ea3afbf99..000000000
--- a/scrapy/addons/builtins.py
+++ /dev/null
@@ -1,146 +0,0 @@
-import scrapy
-from scrapy.addons import Addon
-
-__all__ = [
- "make_builtin_addon",
- "depth",
- "httperror",
- "offsite",
- "referer",
- "urllength",
- "ajaxcrawl",
- "chunked",
- "cookies",
- "defaultheaders",
- "downloadtimeout",
- "httpauth",
- "httpcache",
- "httpcompression",
- "httpproxy",
- "metarefresh",
- "redirect",
- "retry",
- "robotstxt",
- "stats",
- "useragent",
- "autothrottle",
- "corestats",
- "closespider",
- "debugger",
- "feedexport",
- "logstats",
- "memdebug",
- "memusage",
- "spiderstate",
- "stacktracedump",
- "statsmailer",
- "telnetconsole",
-]
-
-
-def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):
- class ThisAddon(Addon):
- name = addon_name
- version = addon_version or scrapy.__version__
- default_config = addon_default_config or {}
-
- return ThisAddon
-
-
-# XXX: Below are CLASSES that have lowercase names. This is in line with the
-# original SEP-021 but violates PEP8.
-# We might consider prepending all built-in addon names with scrapy_ or similar
-# to reduce the chance of name clashes.
-
-# SPIDER MIDDLEWARES
-
-depth = make_builtin_addon("depth")
-
-httperror = make_builtin_addon("httperror")
-
-offsite = make_builtin_addon("offsite")
-
-referer = make_builtin_addon("referer")
-
-urllength = make_builtin_addon("urllength")
-
-
-# DOWNLOADER MIDDLEWARES
-
-ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})
-
-chunked = make_builtin_addon("chunked")
-
-cookies = make_builtin_addon("cookies")
-
-defaultheaders = make_builtin_addon("defaultheaders")
-
-
-# Assume every config entry is a header
-def defaultheaders_export_config(self, config, settings):
- conf = self.default_config or {}
- conf.update(config)
- settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")
-
-
-defaultheaders.export_config = defaultheaders_export_config
-
-downloadtimeout = make_builtin_addon("downloadtimeout")
-downloadtimeout.config_mapping = {
- "timeout": "DOWNLOAD_TIMEOUT",
- "download_timeout": "DOWNLOAD_TIMEOUT",
-}
-
-httpauth = make_builtin_addon("httpauth")
-
-httpcache = make_builtin_addon("httpcache", {"enabled": True})
-
-httpcompression = make_builtin_addon("httpcompression")
-httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}
-
-httpproxy = make_builtin_addon("httpproxy")
-
-metarefresh = make_builtin_addon("metarefresh")
-metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}
-
-redirect = make_builtin_addon("redirect")
-
-retry = make_builtin_addon("retry")
-
-robotstxt = make_builtin_addon("robotstxt", {"obey": True})
-
-stats = make_builtin_addon("stats")
-
-useragent = make_builtin_addon("useragent")
-useragent.config_mapping = {"user_agent": "USER_AGENT"}
-
-
-# ITEM PIPELINES
-
-
-# EXTENSIONS
-
-autothrottle = make_builtin_addon("autothrottle", {"enabled": True})
-
-corestats = make_builtin_addon("corestats")
-
-closespider = make_builtin_addon("closespider")
-
-debugger = make_builtin_addon("debugger")
-
-feedexport = make_builtin_addon("feedexport")
-feedexport.settings_prefix = "FEED"
-
-logstats = make_builtin_addon("logstats")
-
-memdebug = make_builtin_addon("memdebug", {"enabled": True})
-
-memusage = make_builtin_addon("memusage", {"enabled": True})
-
-spiderstate = make_builtin_addon("spiderstate")
-
-stacktracedump = make_builtin_addon("stacktracedump")
-
-statsmailer = make_builtin_addon("statsmailer")
-
-telnetconsole = make_builtin_addon("telnetconsole")
diff --git a/sep/sep-021.rst b/sep/sep-021.rst
index cb1701014..47cba004c 100644
--- a/sep/sep-021.rst
+++ b/sep/sep-021.rst
@@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons``
submodule). If the object found has an ``_addon`` attribute, that attribute
will be treated as the found add-on. This allows, for example, to change the
add-on based on the Python version.
-
-Updating existing extensions
-----------------------------
-
-An ``Addon`` class is introduced that add-on developers may or may not subclass
-depending on how much of the 'default functionality' they want. Naturally, it
-does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()``
-exposes the add-on configuration into the global settings namespace with an
-appropriate name, e.g. this section from ``scrapy.cfg``::
-
- [httpcache]
- dir = /some/dir
-
-would expose ``HTTPCACHE_DIR``.
-
-Add-on modules will be written for all built-in extensions and placed in
-``scrapy.addons``. For many default Scrapy components, it will be sufficient to
-create a subclass of ``Addon`` with minor or no method modifications. The
-component code remains where it is (i.e. in ``scrapy.pipelines``, etc.).
-
-Later, the global settings namespace could be cleaned up in a backwards
--incompatible fashion by deprecating support for the global setting names, e.g.
-``HTTPCACHE_DIR``, and instead instantiate the components with the add-on
-configuration in ``update_settings()``.
-
diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py
deleted file mode 100644
index 1050cbbed..000000000
--- a/tests/test_addons/test_builtins.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import unittest
-
-import scrapy
-import scrapy.addons
-from scrapy.addons.builtins import make_builtin_addon
-from scrapy.settings import Settings
-
-
-class BuiltinAddonsTest(unittest.TestCase):
- def test_make_builtin_addon(self):
- httpcache = make_builtin_addon("httpcache", {"enabled": True})
- self.assertEqual(httpcache.name, "httpcache")
- self.assertEqual(httpcache.default_config, {"enabled": True})
- self.assertEqual(httpcache.version, scrapy.__version__)
- httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")
- self.assertEqual(httpcache.version, "99.9")
-
- def test_defaultheaders_export_config(self):
- settings = Settings()
- dh = scrapy.addons.defaultheaders()
- dh.export_config({"X-Test-Header": "val"}, settings)
- self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])
- self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")