Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Test, linter etc. fixes.

Andrey Rakhmatullin075ad6f196e7
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 4dab15a2a..ba6e839a5 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see
Provide add-on interface through a module:
-.. No idea why just using '::' doesn't work for this one
.. code-block:: python
- name = 'AddonModule'
- version = '1.0'
+ name = "AddonModule"
+ version = "1.0"
+
class MyPipeline(object):
- # ...
+ ...
+
class MyDownloaderMiddleware(object):
- # ...
+ ...
+
def update_settings(config, settings):
settings.set(
- 'ITEM_PIPELINES',
+ "ITEM_PIPELINES",
{MyPipeline(): 200},
- priority='addon',
- }
+ priority="addon",
+ )
settings.set(
- 'DOWNLOADER_MIDDLEWARES',
+ "DOWNLOADER_MIDDLEWARES",
{MyDownloaderMiddleware(): 800},
- priority='addon',
- }
+ priority="addon",
+ )
Forward to other add-ons depending on Python version::
diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py
index 169c79eac..aad9ef789 100644
--- a/scrapy/addons/__init__.py
+++ b/scrapy/addons/__init__.py
@@ -1,10 +1,10 @@
-from collections import defaultdict, Mapping, OrderedDict
-from inspect import isclass
-import six
import warnings
+from collections import OrderedDict, defaultdict
+from collections.abc import Mapping
+from inspect import isclass
-from pkg_resources import WorkingSet, Distribution, Requirement
import zope.interface
+from pkg_resources import Distribution, Requirement, WorkingSet
from zope.interface.verify import verifyObject
from scrapy.interfaces import IAddon
@@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object
@zope.interface.implementer(IAddon)
class Addon(object):
-
basic_settings = None
"""``dict`` of settings that will be exported via :meth:`export_basics`."""
@@ -83,8 +82,8 @@ class Addon(object):
else:
# e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100}
k = comp
- v = config.get('order', self.component_order)
- settings[self.component_type].update({k: v}, 'addon')
+ v = config.get("order", self.component_order)
+ settings[self.component_type].update({k: v}, "addon")
def export_basics(self, settings):
"""Export the :attr:`basic_settings` attribute into the settings object.
@@ -95,8 +94,8 @@ class Addon(object):
:param settings: Settings object into which to expose the basic settings
:type settings: :class:`~scrapy.settings.Settings`
"""
- for setting, value in six.iteritems(self.basic_settings or {}):
- settings.set(setting, value, 'addon')
+ for setting, value in (self.basic_settings or {}).items():
+ settings.set(setting, value, "addon")
def export_config(self, config, settings):
"""Export the add-on configuration, all keys in caps and with
@@ -121,14 +120,13 @@ class Addon(object):
prefix = self.settings_prefix or self.name
# Since default exported config is case-insensitive (everything will be
# uppercased), make mapped config case-insensitive as well
- conf_mapping = {k.lower(): v
- for k, v in six.iteritems(self.config_mapping or {})}
- for key, val in six.iteritems(conf):
+ conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}
+ for key, val in conf.items():
if key.lower() in conf_mapping:
key = conf_mapping[key.lower()]
else:
- key = (prefix + '_' + key).upper()
- settings.set(key, val, 'addon')
+ key = (prefix + "_" + key).upper()
+ settings.set(key, val, "addon")
def update_settings(self, config, settings):
"""Export both the basic settings and the add-on configuration. I.e.,
@@ -210,11 +208,11 @@ class AddonManager(Mapping):
verifyObject(IAddon, addon)
name = addon.name
if name in self:
- raise ValueError("Addon '{}' already loaded".format(name))
+ raise ValueError(f"Addon '{name}' already loaded")
self._addons[name] = addon
self.configs[name] = config or {}
if name in self._disable_on_add:
- self.configs[name]['_enabled'] = False
+ self.configs[name]["_enabled"] = False
self._disable_on_add.remove(name)
def remove(self, addon):
@@ -229,7 +227,7 @@ class AddonManager(Mapping):
"""
if addon in self:
del self[addon]
- elif hasattr(addon, 'name') and addon.name in self:
+ elif hasattr(addon, "name") and addon.name in self:
del self[addon.name]
else:
try:
@@ -250,14 +248,14 @@ class AddonManager(Mapping):
:param path: Python or file path to an add-on
:type path: ``str``
"""
- if isinstance(path, six.string_types):
+ if isinstance(path, str):
try:
obj = load_module_or_object(path)
except NameError:
- raise NameError("Could not find add-on '%s'" % path)
+ raise NameError(f"Could not find add-on '{path}'")
else:
obj = path
- if hasattr(obj, '_addon'):
+ if hasattr(obj, "_addon"):
obj = AddonManager.get_addon(obj._addon)
return obj
@@ -284,7 +282,7 @@ class AddonManager(Mapping):
and values correspond to their configuration
:type addonsdict: ``dict``
"""
- for addonpath, addoncfg in six.iteritems(addonsdict):
+ for addonpath, addoncfg in addonsdict.items():
self.add(addonpath, addoncfg)
def load_settings(self, settings):
@@ -299,7 +297,7 @@ class AddonManager(Mapping):
which to read the add-on configuration
:type settings: :class:`~scrapy.settings.Settings`
"""
- paths = build_component_list(settings['ADDONS'])
+ paths = build_component_list(settings["ADDONS"])
addons = [self.get_addon(path) for path in paths]
configs = [settings.getdict(addon.name.upper()) for addon in addons]
for a, c in zip(addons, configs):
@@ -322,20 +320,23 @@ class AddonManager(Mapping):
add-on.
"""
# Collect all active add-ons and the components they provide
- ws = WorkingSet('')
+ ws = WorkingSet("")
def add_dist(project_name, version, **kwargs):
- if project_name in ws.entry_keys.get('scrapy', []):
- raise ImportError("Component {} provided by multiple add-ons"
- "".format(project_name))
+ if project_name in ws.entry_keys.get("scrapy", []):
+ raise ImportError(
+ f"Component {project_name} provided by multiple add-ons"
+ )
else:
- dist = Distribution(project_name=project_name, version=version,
- **kwargs)
- ws.add(dist, entry='scrapy')
+ dist = Distribution(
+ project_name=project_name, version=version, **kwargs
+ )
+ ws.add(dist, entry="scrapy")
+
for name in self:
ver = self[name].version
add_dist(name, ver)
- for provides_name in getattr(self[name], 'provides', []):
+ for provides_name in getattr(self[name], "provides", []):
add_dist(provides_name, ver)
# Collect all required and modified components
@@ -345,8 +346,9 @@ class AddonManager(Mapping):
for entry in getattr(self[name], attribute_name, []):
attrs[entry].append(name)
return attrs
- modified = compile_attribute_dict('modifies')
- required = compile_attribute_dict('requires')
+
+ modified = compile_attribute_dict("modifies")
+ required = compile_attribute_dict("requires")
req_or_mod = set(required.keys()).union(modified.keys())
for reqstr in req_or_mod:
@@ -355,15 +357,16 @@ class AddonManager(Mapping):
# our own exception or is it helpful enough?
if ws.find(req) is None:
raise ImportError(
- "Add-ons {} require or modify missing component {}"
- "".format(required[reqstr]+modified[reqstr], reqstr)
+ f"Add-ons {required[reqstr] + modified[reqstr]} require"
+ f" or modify missing component {reqstr}"
)
mod_and_req = set(required.keys()).intersection(modified.keys())
for conflict in mod_and_req:
- warnings.warn("Component '{}', required by add-ons {}, is modified "
- "by add-ons {}".format(conflict, required[conflict],
- modified[conflict]))
+ warnings.warn(
+ f"Component '{conflict}', required by add-ons {required[conflict]},"
+ f" is modified by add-ons {modified[conflict]}"
+ )
def disable(self, addon):
"""Disable an add-on, i.e. prevent its callbacks from being called.
@@ -375,7 +378,7 @@ class AddonManager(Mapping):
:type addon: ``str``
"""
if addon in self:
- self.configs[addon]['_enabled'] = False
+ self.configs[addon]["_enabled"] = False
else:
self._disable_on_add.append(addon)
@@ -389,23 +392,23 @@ class AddonManager(Mapping):
:type addon: ``str``
"""
if addon in self:
- self.configs[addon]['_enabled'] = True
+ self.configs[addon]["_enabled"] = True
elif addon in self._disable_on_add:
self._disable_on_add.remove(addon)
else:
- raise ValueError("Add-ons need to be added before they can be "
- "enabled")
+ raise ValueError("Add-ons need to be added before they can be " "enabled")
@property
def disabled(self):
"""Names of disabled add-ons"""
- return ([a for a in self if not self.configs[a].get('_enabled', True)] +
- self._disable_on_add)
+ return [
+ a for a in self if not self.configs[a].get("_enabled", True)
+ ] + self._disable_on_add
@property
def enabled(self):
"""Names of enabled add-ons"""
- return [a for a in self if self.configs[a].get('_enabled', True)]
+ return [a for a in self if self.configs[a].get("_enabled", True)]
def _call_if_exists(self, obj, cbname, *args, **kwargs):
if obj is None:
@@ -418,9 +421,10 @@ class AddonManager(Mapping):
cb(*args, **kwargs)
def _call_addon(self, addonname, cbname, *args, **kwargs):
- if self.configs[addonname].get('_enabled', True):
- self._call_if_exists(self[addonname], cbname,
- self.configs[addonname], *args, **kwargs)
+ if self.configs[addonname].get("_enabled", True):
+ self._call_if_exists(
+ self[addonname], cbname, self.configs[addonname], *args, **kwargs
+ )
def update_addons(self):
"""Call ``update_addons()`` of all held add-ons.
@@ -432,7 +436,7 @@ class AddonManager(Mapping):
while called_addons != set(self):
for name in set(self).difference(called_addons):
called_addons.add(name)
- self._call_addon(name, 'update_addons', self)
+ self._call_addon(name, "update_addons", self)
def update_settings(self, settings):
"""Call ``update_settings()`` of all held add-ons.
@@ -442,7 +446,7 @@ class AddonManager(Mapping):
:type settings: :class:`~scrapy.settings.Settings`
"""
for name in self:
- self._call_addon(name, 'update_settings', settings)
+ self._call_addon(name, "update_settings", settings)
def check_configuration(self, crawler):
"""Call ``check_configuration()`` of all held add-ons.
@@ -451,7 +455,7 @@ class AddonManager(Mapping):
:type crawler: :class:`~scrapy.crawler.Crawler`
"""
for name in self:
- self._call_addon(name, 'check_configuration', crawler)
+ self._call_addon(name, "check_configuration", crawler)
-from scrapy.addons.builtins import *
+from scrapy.addons.builtins import * # noqa
diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py
index 9babdeb6f..ea3afbf99 100644
--- a/scrapy/addons/builtins.py
+++ b/scrapy/addons/builtins.py
@@ -1,23 +1,44 @@
import scrapy
from scrapy.addons import Addon
-__all__ = ['make_builtin_addon',
-
- 'depth', 'httperror', 'offsite', 'referer', 'urllength',
-
- 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders',
- 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression',
- 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt',
- 'stats', 'useragent',
-
- 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport',
- 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump',
- 'statsmailer', 'telnetconsole',
- ]
-
-
-def make_builtin_addon(addon_name, addon_default_config=None,
- addon_version=None):
+__all__ = [
+ "make_builtin_addon",
+ "depth",
+ "httperror",
+ "offsite",
+ "referer",
+ "urllength",
+ "ajaxcrawl",
+ "chunked",
+ "cookies",
+ "defaultheaders",
+ "downloadtimeout",
+ "httpauth",
+ "httpcache",
+ "httpcompression",
+ "httpproxy",
+ "metarefresh",
+ "redirect",
+ "retry",
+ "robotstxt",
+ "stats",
+ "useragent",
+ "autothrottle",
+ "corestats",
+ "closespider",
+ "debugger",
+ "feedexport",
+ "logstats",
+ "memdebug",
+ "memusage",
+ "spiderstate",
+ "stacktracedump",
+ "statsmailer",
+ "telnetconsole",
+]
+
+
+def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):
class ThisAddon(Addon):
name = addon_name
version = addon_version or scrapy.__version__
@@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None,
# SPIDER MIDDLEWARES
-depth = make_builtin_addon('depth')
+depth = make_builtin_addon("depth")
-httperror = make_builtin_addon('httperror')
+httperror = make_builtin_addon("httperror")
-offsite = make_builtin_addon('offsite')
+offsite = make_builtin_addon("offsite")
-referer = make_builtin_addon('referer')
+referer = make_builtin_addon("referer")
-urllength = make_builtin_addon('urllength')
+urllength = make_builtin_addon("urllength")
# DOWNLOADER MIDDLEWARES
-ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True})
+ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})
-chunked = make_builtin_addon('chunked')
+chunked = make_builtin_addon("chunked")
+
+cookies = make_builtin_addon("cookies")
+
+defaultheaders = make_builtin_addon("defaultheaders")
-cookies = make_builtin_addon('cookies')
-defaultheaders = make_builtin_addon('defaultheaders')
# Assume every config entry is a header
def defaultheaders_export_config(self, config, settings):
conf = self.default_config or {}
conf.update(config)
- settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon')
+ settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")
+
+
defaultheaders.export_config = defaultheaders_export_config
-downloadtimeout = make_builtin_addon('downloadtimeout')
-downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT',
- 'download_timeout': 'DOWNLOAD_TIMEOUT'}
+downloadtimeout = make_builtin_addon("downloadtimeout")
+downloadtimeout.config_mapping = {
+ "timeout": "DOWNLOAD_TIMEOUT",
+ "download_timeout": "DOWNLOAD_TIMEOUT",
+}
-httpauth = make_builtin_addon('httpauth')
+httpauth = make_builtin_addon("httpauth")
-httpcache = make_builtin_addon('httpcache', {'enabled': True})
+httpcache = make_builtin_addon("httpcache", {"enabled": True})
-httpcompression = make_builtin_addon('httpcompression')
-httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'}
+httpcompression = make_builtin_addon("httpcompression")
+httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}
-httpproxy = make_builtin_addon('httpproxy')
+httpproxy = make_builtin_addon("httpproxy")
-metarefresh = make_builtin_addon('metarefresh')
-metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'}
+metarefresh = make_builtin_addon("metarefresh")
+metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}
-redirect = make_builtin_addon('redirect')
+redirect = make_builtin_addon("redirect")
-retry = make_builtin_addon('retry')
+retry = make_builtin_addon("retry")
-robotstxt = make_builtin_addon('robotstxt', {'obey': True})
+robotstxt = make_builtin_addon("robotstxt", {"obey": True})
-stats = make_builtin_addon('stats')
+stats = make_builtin_addon("stats")
-useragent = make_builtin_addon('useragent')
-useragent.config_mapping = {'user_agent': 'USER_AGENT'}
+useragent = make_builtin_addon("useragent")
+useragent.config_mapping = {"user_agent": "USER_AGENT"}
# ITEM PIPELINES
@@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'}
# EXTENSIONS
-autothrottle = make_builtin_addon('autothrottle', {'enabled': True})
+autothrottle = make_builtin_addon("autothrottle", {"enabled": True})
-corestats = make_builtin_addon('corestats')
+corestats = make_builtin_addon("corestats")
-closespider = make_builtin_addon('closespider')
+closespider = make_builtin_addon("closespider")
-debugger = make_builtin_addon('debugger')
+debugger = make_builtin_addon("debugger")
-feedexport = make_builtin_addon('feedexport')
-feedexport.settings_prefix = 'FEED'
+feedexport = make_builtin_addon("feedexport")
+feedexport.settings_prefix = "FEED"
-logstats = make_builtin_addon('logstats')
+logstats = make_builtin_addon("logstats")
-memdebug = make_builtin_addon('memdebug', {'enabled': True})
+memdebug = make_builtin_addon("memdebug", {"enabled": True})
-memusage = make_builtin_addon('memusage', {'enabled': True})
+memusage = make_builtin_addon("memusage", {"enabled": True})
-spiderstate = make_builtin_addon('spiderstate')
+spiderstate = make_builtin_addon("spiderstate")
-stacktracedump = make_builtin_addon('stacktracedump')
+stacktracedump = make_builtin_addon("stacktracedump")
-statsmailer = make_builtin_addon('statsmailer')
+statsmailer = make_builtin_addon("statsmailer")
-telnetconsole = make_builtin_addon('telnetconsole')
+telnetconsole = make_builtin_addon("telnetconsole")
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 4bf7e7e66..8577cce02 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -85,7 +85,7 @@ def load_module_or_object(path):
return load_object(path)
except (ValueError, NameError, ImportError):
pass
- raise NameError("Could not load '%s'" % path)
+ raise NameError(f"Could not load '{path}'")
def walk_modules(path):
diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py
index a4e278fa5..741dd81cf 100644
--- a/tests/test_addons/__init__.py
+++ b/tests/test_addons/__init__.py
@@ -1,145 +1,143 @@
-from collections import OrderedDict
import itertools
-import os.path
-import six
-from tests import mock
import unittest
import warnings
+from collections import OrderedDict
+from unittest import mock
from pkg_resources import VersionConflict
-import zope.interface
+from zope.interface import directlyProvides
+from zope.interface.exceptions import BrokenImplementation, MultipleInvalid
from zope.interface.verify import verifyObject
-from zope.interface.exceptions import BrokenImplementation
from scrapy.addons import Addon, AddonManager
from scrapy.crawler import Crawler
from scrapy.interfaces import IAddon
from scrapy.settings import BaseSettings
-from . import addons
-from . import addonmod
+from . import addonmod, addons
class AddonTest(unittest.TestCase):
-
def setUp(self):
self.rawaddon = Addon()
class AddonWithAttributes(Addon):
- name = 'Test'
- version = '1.0'
+ name = "Test"
+ version = "1.0"
+
self.testaddon = AddonWithAttributes()
def test_interface(self):
# Raw Addon should fail exactly b/c name and version are not given
- self.assertFalse(hasattr(self.rawaddon, 'name'))
- self.assertFalse(hasattr(self.rawaddon, 'version'))
- self.assertRaises(BrokenImplementation, verifyObject, IAddon,
- self.rawaddon)
+ self.assertFalse(hasattr(self.rawaddon, "name"))
+ self.assertFalse(hasattr(self.rawaddon, "version"))
+ self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon)
verifyObject(IAddon, self.testaddon)
def test_export_component(self):
- settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(),
- 'DOWNLOAD_HANDLERS': BaseSettings()},
- 'default')
+ settings = BaseSettings(
+ {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()},
+ "default",
+ )
self.testaddon.component_type = None
self.testaddon.export_component({}, settings)
- self.assertEqual(len(settings['ITEM_PIPELINES']), 0)
- self.testaddon.component_type = 'ITEM_PIPELINES'
- self.testaddon.component = 'test.component'
+ self.assertEqual(len(settings["ITEM_PIPELINES"]), 0)
+ self.testaddon.component_type = "ITEM_PIPELINES"
+ self.testaddon.component = "test.component"
self.testaddon.export_component({}, settings)
- six.assertCountEqual(self, settings['ITEM_PIPELINES'],
- ['test.component'])
- self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0)
+ self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"])
+ self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0)
self.testaddon.component_order = 313
self.testaddon.export_component({}, settings)
- self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313)
- self.testaddon.component_type = 'DOWNLOAD_HANDLERS'
- self.testaddon.component_key = 'http'
+ self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313)
+ self.testaddon.component_type = "DOWNLOAD_HANDLERS"
+ self.testaddon.component_key = "http"
self.testaddon.export_component({}, settings)
- self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'],
- 'test.component')
+ self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component")
def test_export_basics(self):
settings = BaseSettings()
- self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True}
+ self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True}
self.testaddon.export_basics(settings)
- self.assertEqual(settings['TESTKEY'], 313)
- self.assertEqual(settings['OTHERKEY'], True)
- self.assertEqual(settings.getpriority('TESTKEY'), 15)
+ self.assertEqual(settings["TESTKEY"], 313)
+ self.assertEqual(settings["OTHERKEY"], True)
+ self.assertEqual(settings.getpriority("TESTKEY"), 15)
def test_export_config(self):
settings = BaseSettings()
self.testaddon.settings_prefix = None
- self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'}
- self.testaddon.default_config = {'key': 55, 'defaultkey': 100}
- self.testaddon.export_config({'key': 313, 'OTHERKEY': True,
- 'mapped_KEY': 99}, settings)
- self.assertEqual(settings['TEST_KEY'], 313)
- self.assertEqual(settings['TEST_DEFAULTKEY'], 100)
- self.assertEqual(settings['TEST_OTHERKEY'], True)
- self.assertNotIn('MAPPED_key', settings)
- self.assertNotIn('MAPPED_KEY', settings)
- self.assertEqual(settings['MAPPING_WORKED'], 99)
- self.assertEqual(settings.getpriority('TEST_KEY'), 15)
-
- self.testaddon.settings_prefix = 'PREF'
- self.testaddon.export_config({'newkey': 99}, settings)
- self.assertEqual(settings['PREF_NEWKEY'], 99)
-
- with mock.patch.object(settings, 'set') as mock_set:
+ self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}
+ self.testaddon.default_config = {"key": 55, "defaultkey": 100}
+ self.testaddon.export_config(
+ {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings
+ )
+ self.assertEqual(settings["TEST_KEY"], 313)
+ self.assertEqual(settings["TEST_DEFAULTKEY"], 100)
+ self.assertEqual(settings["TEST_OTHERKEY"], True)
+ self.assertNotIn("MAPPED_key", settings)
+ self.assertNotIn("MAPPED_KEY", settings)
+ self.assertEqual(settings["MAPPING_WORKED"], 99)
+ self.assertEqual(settings.getpriority("TEST_KEY"), 15)
+
+ self.testaddon.settings_prefix = "PREF"
+ self.testaddon.export_config({"newkey": 99}, settings)
+ self.assertEqual(settings["PREF_NEWKEY"], 99)
+
+ with mock.patch.object(settings, "set") as mock_set:
self.testaddon.settings_prefix = False
- self.testaddon.export_config({'thirdnewkey': 99}, settings)
+ self.testaddon.export_config({"thirdnewkey": 99}, settings)
self.assertEqual(mock_set.call_count, 0)
def test_update_settings(self):
settings = BaseSettings()
- settings.set('TEST_KEY1', 'default', priority='default')
- settings.set('TEST_KEY2', 'project', priority='project')
+ settings.set("TEST_KEY1", "default", priority="default")
+ settings.set("TEST_KEY2", "project", priority="project")
self.testaddon.settings_prefix = None
- self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'}
- addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'}
+ self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"}
+ addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}
self.testaddon.update_settings(addon_config, settings)
- self.assertEqual(settings['OTHERTEST_KEY'], 'addon')
- self.assertEqual(settings['TEST_KEY1'], 'addon')
- self.assertEqual(settings['TEST_KEY2'], 'project')
- self.assertEqual(settings['TEST_KEY3'], 'addon')
+ self.assertEqual(settings["OTHERTEST_KEY"], "addon")
+ self.assertEqual(settings["TEST_KEY1"], "addon")
+ self.assertEqual(settings["TEST_KEY2"], "project")
+ self.assertEqual(settings["TEST_KEY3"], "addon")
class AddonManagerTest(unittest.TestCase):
-
def setUp(self):
self.manager = AddonManager()
def test_add(self):
manager = AddonManager()
- manager.add(addonmod, {'key': 'val1'})
- manager.add('tests.test_addons.addons.GoodAddon')
- six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon'])
- self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
- six.assertCountEqual(self, manager.configs['AddonModule'], ['key'])
- self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
+ manager.add(addonmod, {"key": "val1"})
+ manager.add("tests.test_addons.addons.GoodAddon")
+ self.assertCountEqual(manager, ["AddonModule", "GoodAddon"])
+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
+ self.assertCountEqual(manager.configs["AddonModule"], ["key"])
+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
self.assertRaises(ValueError, manager.add, addonmod)
def test_add_dont_instantiate_providing_classes(self):
class ProviderGoodAddon(addons.GoodAddon):
pass
- zope.interface.directlyProvides(ProviderGoodAddon, IAddon)
+
+ directlyProvides(ProviderGoodAddon, IAddon)
manager = AddonManager()
manager.add(ProviderGoodAddon)
- self.assertIs(manager['GoodAddon'], ProviderGoodAddon)
+ self.assertIs(manager["GoodAddon"], ProviderGoodAddon)
def test_add_verifies(self):
- brokenaddon = self.manager.get_addon(
- 'tests.test_addons.addons.BrokenAddon')
- self.assertRaises(zope.interface.exceptions.BrokenImplementation,
- self.manager.add,
- brokenaddon)
+ brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon")
+ self.assertRaises(
+ BrokenImplementation,
+ self.manager.add,
+ brokenaddon,
+ )
def test_add_adds_missing_interface_declaration(self):
class GoodAddonWithoutDeclaration(object):
- name = 'GoodAddonWithoutDeclaration'
- version = '1.0'
+ name = "GoodAddonWithoutDeclaration"
+ version = "1.0"
+
self.manager.add(GoodAddonWithoutDeclaration)
def test_remove(self):
@@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase):
def test_gets_removed(removearg):
manager.add(addonmod)
- self.assertIn('AddonModule', manager)
+ self.assertIn("AddonModule", manager)
manager.remove(removearg)
- self.assertNotIn('AddonModule', manager)
+ self.assertNotIn("AddonModule", manager)
- test_gets_removed('AddonModule')
+ test_gets_removed("AddonModule")
test_gets_removed(addonmod)
- test_gets_removed('tests.test_addons.addonmod')
- self.assertRaises(KeyError, manager.remove, 'nonexistent')
+ test_gets_removed("tests.test_addons.addonmod")
+ self.assertRaises(KeyError, manager.remove, "nonexistent")
self.assertRaises(KeyError, manager.remove, addons.GoodAddon())
def test_get_addon(self):
- goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon')
+ goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon")
self.assertIs(goodaddon, addons.GoodAddon)
loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod")
self.assertIs(loaded_addonmod, addonmod)
- addonspath = os.path.join(os.path.dirname(__file__), 'addons.py')
goodaddon = self.manager.get_addon("tests.test_addons.addons")
self.assertIsInstance(goodaddon, addons.GoodAddon)
- self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent')
+ self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent")
def test_get_addon_forward(self):
class SomeCls(object):
- _addon = 'tests.test_addons.addons.GoodAddon'
+ _addon = "tests.test_addons.addons.GoodAddon"
+
self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon)
def test_get_addon_nested(self):
- x = addons.GoodAddon('outer')
- x._addon = addons.GoodAddon('middle')
- x._addon._addon = addons.GoodAddon('inner')
+ x = addons.GoodAddon("outer")
+ x._addon = addons.GoodAddon("middle")
+ x._addon._addon = addons.GoodAddon("inner")
self.assertIs(self.manager.get_addon(x), x._addon._addon)
def test_load_dict_load_settings(self):
def _test_load_method(func, *args, **kwargs):
manager = AddonManager()
getattr(manager, func)(*args, **kwargs)
- six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule'])
- self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
- six.assertCountEqual(self, manager.configs['GoodAddon'], ['key'])
- self.assertEqual(manager.configs['GoodAddon']['key'], 'val2')
- self.assertEqual(manager['AddonModule'], addonmod)
- self.assertIn('key', manager.configs['AddonModule'])
- self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
+ self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
+ self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
+ self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
+ self.assertEqual(manager["AddonModule"], addonmod)
+ self.assertIn("key", manager.configs["AddonModule"])
+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
addonsdict = {
- "tests.test_addons.addonmod": {'key': 'val1'},
- 'tests.test_addons.addons.GoodAddon': {'key': 'val2'},
- }
- _test_load_method('load_dict', addonsdict)
+ "tests.test_addons.addonmod": {"key": "val1"},
+ "tests.test_addons.addons.GoodAddon": {"key": "val2"},
+ }
+ _test_load_method("load_dict", addonsdict)
settings = BaseSettings()
- settings.set('ADDONS', {"tests.test_addons.addonmod": 0,
- 'tests.test_addons.addons.GoodAddon': 0})
- settings.set('ADDONMODULE', {'key': 'val1'})
- settings.set('GOODADDON', {'key': 'val2'})
- _test_load_method('load_settings', settings)
+ settings.set(
+ "ADDONS",
+ {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0},
+ )
+ settings.set("ADDONMODULE", {"key": "val1"})
+ settings.set("GOODADDON", {"key": "val2"})
+ _test_load_method("load_settings", settings)
def test_load_dict_load_settings_order(self):
def _test_load_method(expected_order, func, *args, **kwargs):
@@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase):
for ordered_addons in itertools.permutations(addonlist):
expected_order = [a.name for a in ordered_addons]
addonsdict = OrderedDict((a, {}) for a in ordered_addons)
- _test_load_method(expected_order, 'load_dict', addonsdict)
- settings = BaseSettings({
- 'ADDONS': {a: i for i, a in enumerate(ordered_addons)}
- })
- _test_load_method(expected_order, 'load_settings', settings)
+ _test_load_method(expected_order, "load_dict", addonsdict)
+ settings = BaseSettings(
+ {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
+ )
+ _test_load_method(expected_order, "load_settings", settings)
def test_enabled_disabled(self):
manager = AddonManager()
- manager.add(addons.GoodAddon('FirstAddon'))
- manager.add(addons.GoodAddon('SecondAddon'))
- self.assertEqual(set(manager.enabled),
- set(('FirstAddon', 'SecondAddon')))
+ manager.add(addons.GoodAddon("FirstAddon"))
+ manager.add(addons.GoodAddon("SecondAddon"))
+ self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))
self.assertEqual(manager.disabled, [])
- manager.disable('FirstAddon')
- self.assertEqual(manager.enabled, ['SecondAddon'])
- self.assertEqual(manager.disabled, ['FirstAddon'])
- manager.enable('FirstAddon')
- self.assertEqual(set(manager.enabled),
- set(('FirstAddon', 'SecondAddon')))
+ manager.disable("FirstAddon")
+ self.assertEqual(manager.enabled, ["SecondAddon"])
+ self.assertEqual(manager.disabled, ["FirstAddon"])
+ manager.enable("FirstAddon")
+ self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))
self.assertEqual(manager.disabled, [])
def test_enable_before_add(self):
manager = AddonManager()
- self.assertRaises(ValueError, manager.enable, 'FirstAddon')
- manager.disable('FirstAddon')
- manager.enable('FirstAddon')
- manager.add(addons.GoodAddon('FirstAddon'))
- self.assertIn('FirstAddon', manager.enabled)
+ self.assertRaises(ValueError, manager.enable, "FirstAddon")
+ manager.disable("FirstAddon")
+ manager.enable("FirstAddon")
+ manager.add(addons.GoodAddon("FirstAddon"))
+ self.assertIn("FirstAddon", manager.enabled)
def test_disable_before_add(self):
manager = AddonManager()
- manager.disable('FirstAddon')
- manager.add(addons.GoodAddon('FirstAddon'))
- self.assertEqual(manager.disabled, ['FirstAddon'])
+ manager.disable("FirstAddon")
+ manager.add(addons.GoodAddon("FirstAddon"))
+ self.assertEqual(manager.disabled, ["FirstAddon"])
def test_callbacks(self):
- first_addon = addons.GoodAddon('FirstAddon')
- second_addon = addons.GoodAddon('SecondAddon')
+ first_addon = addons.GoodAddon("FirstAddon")
+ second_addon = addons.GoodAddon("SecondAddon")
manager = AddonManager()
- manager.add(first_addon, {'test': 'first'})
- manager.add(second_addon, {'test': 'second'})
+ manager.add(first_addon, {"test": "first"})
+ manager.add(second_addon, {"test": "second"})
crawler = mock.create_autospec(Crawler)
settings = BaseSettings()
- with mock.patch.object(first_addon, 'update_addons') as ua_first, \
- mock.patch.object(second_addon, 'update_addons') as ua_second, \
- mock.patch.object(first_addon, 'update_settings') as us_first, \
- mock.patch.object(second_addon, 'update_settings') as us_second, \
- mock.patch.object(first_addon, 'check_configuration') as cc_first, \
- mock.patch.object(second_addon, 'check_configuration') as cc_second:
+ with mock.patch.object(
+ first_addon, "update_addons"
+ ) as ua_first, mock.patch.object(
+ second_addon, "update_addons"
+ ) as ua_second, mock.patch.object(
+ first_addon, "update_settings"
+ ) as us_first, mock.patch.object(
+ second_addon, "update_settings"
+ ) as us_second, mock.patch.object(
+ first_addon, "check_configuration"
+ ) as cc_first, mock.patch.object(
+ second_addon, "check_configuration"
+ ) as cc_second:
manager.update_addons()
- ua_first.assert_called_once_with(manager.configs['FirstAddon'],
- manager)
- ua_second.assert_called_once_with(manager.configs['SecondAddon'],
- manager)
+ ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)
+ ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager)
manager.update_settings(settings)
- us_first.assert_called_once_with(manager.configs['FirstAddon'],
- settings)
- us_second.assert_called_once_with(manager.configs['SecondAddon'],
- settings)
+ us_first.assert_called_once_with(manager.configs["FirstAddon"], settings)
+ us_second.assert_called_once_with(manager.configs["SecondAddon"], settings)
manager.check_configuration(crawler)
- cc_first.assert_called_once_with(manager.configs['FirstAddon'],
- crawler)
- cc_second.assert_called_once_with(manager.configs['SecondAddon'],
- crawler)
+ cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler)
+ cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler)
self.assertEqual(ua_first.call_count, 1)
self.assertEqual(ua_second.call_count, 1)
self.assertEqual(us_first.call_count, 1)
@@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase):
us_first.reset_mock()
us_second.reset_mock()
- manager.disable('FirstAddon')
+ manager.disable("FirstAddon")
manager.update_settings(settings)
self.assertEqual(us_first.call_count, 0)
- manager.enable('FirstAddon')
+ manager.enable("FirstAddon")
manager.update_settings(settings)
self.assertEqual(us_first.call_count, 1)
self.assertEqual(us_second.call_count, 2)
@@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase):
# This will become relevant when we let spiders implement the add-on
# interface and should be replaced with a test where
# AddonManager.spidercls = None then.
- manager._call_if_exists(None, 'irrelevant')
+ manager._call_if_exists(None, "irrelevant")
def test_update_addons_last_minute_add(self):
class AddedAddon(addons.GoodAddon):
- name = 'AddedAddon'
+ name = "AddedAddon"
class FirstAddon(addons.GoodAddon):
- name = 'FirstAddon'
+ name = "FirstAddon"
def update_addons(self, config, addons):
addons.add(AddedAddon())
manager = AddonManager()
first_addon = FirstAddon()
- with mock.patch.object(first_addon, 'update_addons',
- wraps=first_addon.update_addons) as ua_first, \
- mock.patch.object(AddedAddon, 'update_addons') as ua_added:
- manager.add(first_addon, {'non-empty': 'dict'})
+ with mock.patch.object(
+ first_addon, "update_addons", wraps=first_addon.update_addons
+ ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added:
+ manager.add(first_addon, {"non-empty": "dict"})
manager.update_addons()
- six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon'])
- ua_first.assert_called_once_with(manager.configs['FirstAddon'],
- manager)
- ua_added.assert_called_once_with(manager.configs['AddedAddon'],
- manager)
+ self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"])
+ ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)
+ ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager)
def test_check_dependency_clashes_attributes(self):
provides = addons.GoodAddon("ProvidesAddon")
- provides.provides = ('test', )
+ provides.provides = ("test",)
provides2 = addons.GoodAddon("ProvidesAddon2")
- provides2.provides = ('test', )
+ provides2.provides = ("test",)
requires = addons.GoodAddon("RequiresAddon")
- requires.requires = ('test', )
+ requires.requires = ("test",)
requires_name = addons.GoodAddon("RequiresNameAddon")
- requires_name.requires = ('ProvidesAddon', )
+ requires_name.requires = ("ProvidesAddon",)
requires_newer = addons.GoodAddon("RequiresNewerAddon")
- requires_newer.requires = ('test>=2.0', )
+ requires_newer.requires = ("test>=2.0",)
modifies = addons.GoodAddon("ModifiesAddon")
- modifies.modifies = ('test', )
+ modifies.modifies = ("test",)
def check_with(*addons):
manager = AddonManager()
diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py
index 8ecf4b81d..c59f6737f 100644
--- a/tests/test_addons/addonmod.py
+++ b/tests/test_addons/addonmod.py
@@ -9,8 +9,10 @@ FROM = "test_addons.addonmod"
name = "AddonModule"
version = "1.0"
+
def update_settings(config, settings):
pass
+
def check_configuration(config, crawler):
pass
diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py
index f3442b192..4adb9fe8f 100644
--- a/tests/test_addons/addons.py
+++ b/tests/test_addons/addons.py
@@ -1,18 +1,16 @@
import zope.interface
-from scrapy.addons import Addon
from scrapy.interfaces import IAddon
class Addon(object):
- FROM = 'test_addons.addons'
+ FROM = "test_addons.addons"
@zope.interface.declarations.implementer(IAddon)
class GoodAddon(object):
-
- name = 'GoodAddon'
- version = '1.0'
+ name = "GoodAddon"
+ version = "1.0"
def __init__(self, name=None, version=None):
if name is not None:
@@ -32,8 +30,7 @@ class GoodAddon(object):
@zope.interface.declarations.implementer(IAddon)
class BrokenAddon(object):
-
- name = 'BrokenAddon'
+ name = "BrokenAddon"
# No version
diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py
index c89876950..1050cbbed 100644
--- a/tests/test_addons/test_builtins.py
+++ b/tests/test_addons/test_builtins.py
@@ -7,19 +7,17 @@ from scrapy.settings import Settings
class BuiltinAddonsTest(unittest.TestCase):
-
def test_make_builtin_addon(self):
- httpcache = make_builtin_addon('httpcache', {'enabled': True})
- self.assertEqual(httpcache.name, 'httpcache')
- self.assertEqual(httpcache.default_config, {'enabled': True})
+ httpcache = make_builtin_addon("httpcache", {"enabled": True})
+ self.assertEqual(httpcache.name, "httpcache")
+ self.assertEqual(httpcache.default_config, {"enabled": True})
self.assertEqual(httpcache.version, scrapy.__version__)
- httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9')
- self.assertEqual(httpcache.version, '99.9')
+ httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")
+ self.assertEqual(httpcache.version, "99.9")
def test_defaultheaders_export_config(self):
settings = Settings()
dh = scrapy.addons.defaultheaders()
- dh.export_config({'X-Test-Header': 'val'}, settings)
- self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS'])
- self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'],
- 'val')
+ dh.export_config({"X-Test-Header": "val"}, settings)
+ self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])
+ self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index e353f80cb..1e97863b0 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -419,7 +419,7 @@ with multiples lines
addonmgr = AddonManager()
addonmgr.add(FailedCheckAddon())
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
crawler.addons = addonmgr
# Doesn't work in 'precise' test environment:
# with self.assertRaises(ValueError):
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 7ddf952f7..fd57d846e 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_instances_from_settings(self):
settings = Settings()
- myM3 = M3()
class InstanceTestMiddlewareManager(MiddlewareManager):
@classmethod
def _get_mwlist_from_settings(cls, settings):
- return ["tests.test_middleware.M1", M2, myM3]
+ return ["tests.test_middleware.M1", M2]
mwman = InstanceTestMiddlewareManager.from_settings(settings)
self.assertIsInstance(mwman.middlewares[0], M1)
self.assertIsInstance(mwman.middlewares[1], M2)
- self.assertIs(mwman.middlewares[2], myM3)