Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove builtin addons.
Andrey Rakhmatullin55ac26228b85
diff --git a/scrapy/addons/__init__.py b/scrapy/addons.pysimilarity index 99%rename from scrapy/addons/__init__.pyrename to scrapy/addons.pyindex aad9ef789..c9b873039 100644--- a/scrapy/addons/__init__.py+++ b/scrapy/addons.py@@ -456,6 +456,3 @@ class AddonManager(Mapping):"""for name in self:self._call_addon(name, "check_configuration", crawler)---from scrapy.addons.builtins import * # noqadiff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.pydeleted file mode 100644index ea3afbf99..000000000--- a/scrapy/addons/builtins.py+++ /dev/null@@ -1,146 +0,0 @@-import scrapy-from scrapy.addons import Addon--__all__ = [- "make_builtin_addon",- "depth",- "httperror",- "offsite",- "referer",- "urllength",- "ajaxcrawl",- "chunked",- "cookies",- "defaultheaders",- "downloadtimeout",- "httpauth",- "httpcache",- "httpcompression",- "httpproxy",- "metarefresh",- "redirect",- "retry",- "robotstxt",- "stats",- "useragent",- "autothrottle",- "corestats",- "closespider",- "debugger",- "feedexport",- "logstats",- "memdebug",- "memusage",- "spiderstate",- "stacktracedump",- "statsmailer",- "telnetconsole",-]---def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):- class ThisAddon(Addon):- name = addon_name- version = addon_version or scrapy.__version__- default_config = addon_default_config or {}-- return ThisAddon---# XXX: Below are CLASSES that have lowercase names. This is in line with the-# original SEP-021 but violates PEP8.-# We might consider prepending all built-in addon names with scrapy_ or similar-# to reduce the chance of name clashes.--# SPIDER MIDDLEWARES--depth = make_builtin_addon("depth")--httperror = make_builtin_addon("httperror")--offsite = make_builtin_addon("offsite")--referer = make_builtin_addon("referer")--urllength = make_builtin_addon("urllength")---# DOWNLOADER MIDDLEWARES--ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})--chunked = make_builtin_addon("chunked")--cookies = make_builtin_addon("cookies")--defaultheaders = make_builtin_addon("defaultheaders")---# Assume every config entry is a header-def defaultheaders_export_config(self, config, settings):- conf = self.default_config or {}- conf.update(config)- settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")---defaultheaders.export_config = defaultheaders_export_config--downloadtimeout = make_builtin_addon("downloadtimeout")-downloadtimeout.config_mapping = {- "timeout": "DOWNLOAD_TIMEOUT",- "download_timeout": "DOWNLOAD_TIMEOUT",-}--httpauth = make_builtin_addon("httpauth")--httpcache = make_builtin_addon("httpcache", {"enabled": True})--httpcompression = make_builtin_addon("httpcompression")-httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}--httpproxy = make_builtin_addon("httpproxy")--metarefresh = make_builtin_addon("metarefresh")-metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}--redirect = make_builtin_addon("redirect")--retry = make_builtin_addon("retry")--robotstxt = make_builtin_addon("robotstxt", {"obey": True})--stats = make_builtin_addon("stats")--useragent = make_builtin_addon("useragent")-useragent.config_mapping = {"user_agent": "USER_AGENT"}---# ITEM PIPELINES---# EXTENSIONS--autothrottle = make_builtin_addon("autothrottle", {"enabled": True})--corestats = make_builtin_addon("corestats")--closespider = make_builtin_addon("closespider")--debugger = make_builtin_addon("debugger")--feedexport = make_builtin_addon("feedexport")-feedexport.settings_prefix = "FEED"--logstats = make_builtin_addon("logstats")--memdebug = make_builtin_addon("memdebug", {"enabled": True})--memusage = make_builtin_addon("memusage", {"enabled": True})--spiderstate = make_builtin_addon("spiderstate")--stacktracedump = make_builtin_addon("stacktracedump")--statsmailer = make_builtin_addon("statsmailer")--telnetconsole = make_builtin_addon("telnetconsole")diff --git a/sep/sep-021.rst b/sep/sep-021.rstindex cb1701014..47cba004c 100644--- a/sep/sep-021.rst+++ b/sep/sep-021.rst@@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons``submodule). If the object found has an ``_addon`` attribute, that attributewill be treated as the found add-on. This allows, for example, to change theadd-on based on the Python version.--Updating existing extensions-------------------------------An ``Addon`` class is introduced that add-on developers may or may not subclass-depending on how much of the 'default functionality' they want. Naturally, it-does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()``-exposes the add-on configuration into the global settings namespace with an-appropriate name, e.g. this section from ``scrapy.cfg``::-- [httpcache]- dir = /some/dir--would expose ``HTTPCACHE_DIR``.--Add-on modules will be written for all built-in extensions and placed in-``scrapy.addons``. For many default Scrapy components, it will be sufficient to-create a subclass of ``Addon`` with minor or no method modifications. The-component code remains where it is (i.e. in ``scrapy.pipelines``, etc.).--Later, the global settings namespace could be cleaned up in a backwards--incompatible fashion by deprecating support for the global setting names, e.g.-``HTTPCACHE_DIR``, and instead instantiate the components with the add-on-configuration in ``update_settings()``.-diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.pydeleted file mode 100644index 1050cbbed..000000000--- a/tests/test_addons/test_builtins.py+++ /dev/null@@ -1,23 +0,0 @@-import unittest--import scrapy-import scrapy.addons-from scrapy.addons.builtins import make_builtin_addon-from scrapy.settings import Settings---class BuiltinAddonsTest(unittest.TestCase):- def test_make_builtin_addon(self):- httpcache = make_builtin_addon("httpcache", {"enabled": True})- self.assertEqual(httpcache.name, "httpcache")- self.assertEqual(httpcache.default_config, {"enabled": True})- self.assertEqual(httpcache.version, scrapy.__version__)- httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")- self.assertEqual(httpcache.version, "99.9")-- def test_defaultheaders_export_config(self):- settings = Settings()- dh = scrapy.addons.defaultheaders()- dh.export_config({"X-Test-Header": "val"}, settings)- self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])- self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")