Scrapy, a fast high-level web crawling & scraping framework for Python.
Test, linter etc. fixes.
Andrey Rakhmatullin075ad6f196e7
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 4dab15a2a..ba6e839a5 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (seeProvide add-on interface through a module:-.. No idea why just using '::' doesn't work for this one.. code-block:: python- name = 'AddonModule'- version = '1.0'+ name = "AddonModule"+ version = "1.0"+class MyPipeline(object):- # ...+ ...+class MyDownloaderMiddleware(object):- # ...+ ...+def update_settings(config, settings):settings.set(- 'ITEM_PIPELINES',+ "ITEM_PIPELINES",{MyPipeline(): 200},- priority='addon',- }+ priority="addon",+ )settings.set(- 'DOWNLOADER_MIDDLEWARES',+ "DOWNLOADER_MIDDLEWARES",{MyDownloaderMiddleware(): 800},- priority='addon',- }+ priority="addon",+ )Forward to other add-ons depending on Python version::diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.pyindex 169c79eac..aad9ef789 100644--- a/scrapy/addons/__init__.py+++ b/scrapy/addons/__init__.py@@ -1,10 +1,10 @@-from collections import defaultdict, Mapping, OrderedDict-from inspect import isclass-import siximport warnings+from collections import OrderedDict, defaultdict+from collections.abc import Mapping+from inspect import isclass-from pkg_resources import WorkingSet, Distribution, Requirementimport zope.interface+from pkg_resources import Distribution, Requirement, WorkingSetfrom zope.interface.verify import verifyObjectfrom scrapy.interfaces import IAddon@@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object@zope.interface.implementer(IAddon)class Addon(object):-basic_settings = None"""``dict`` of settings that will be exported via :meth:`export_basics`."""@@ -83,8 +82,8 @@ class Addon(object):else:# e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100}k = comp- v = config.get('order', self.component_order)- settings[self.component_type].update({k: v}, 'addon')+ v = config.get("order", self.component_order)+ settings[self.component_type].update({k: v}, "addon")def export_basics(self, settings):"""Export the :attr:`basic_settings` attribute into the settings object.@@ -95,8 +94,8 @@ class Addon(object)::param settings: Settings object into which to expose the basic settings:type settings: :class:`~scrapy.settings.Settings`"""- for setting, value in six.iteritems(self.basic_settings or {}):- settings.set(setting, value, 'addon')+ for setting, value in (self.basic_settings or {}).items():+ settings.set(setting, value, "addon")def export_config(self, config, settings):"""Export the add-on configuration, all keys in caps and with@@ -121,14 +120,13 @@ class Addon(object):prefix = self.settings_prefix or self.name# Since default exported config is case-insensitive (everything will be# uppercased), make mapped config case-insensitive as well- conf_mapping = {k.lower(): v- for k, v in six.iteritems(self.config_mapping or {})}- for key, val in six.iteritems(conf):+ conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}+ for key, val in conf.items():if key.lower() in conf_mapping:key = conf_mapping[key.lower()]else:- key = (prefix + '_' + key).upper()- settings.set(key, val, 'addon')+ key = (prefix + "_" + key).upper()+ settings.set(key, val, "addon")def update_settings(self, config, settings):"""Export both the basic settings and the add-on configuration. I.e.,@@ -210,11 +208,11 @@ class AddonManager(Mapping):verifyObject(IAddon, addon)name = addon.nameif name in self:- raise ValueError("Addon '{}' already loaded".format(name))+ raise ValueError(f"Addon '{name}' already loaded")self._addons[name] = addonself.configs[name] = config or {}if name in self._disable_on_add:- self.configs[name]['_enabled'] = False+ self.configs[name]["_enabled"] = Falseself._disable_on_add.remove(name)def remove(self, addon):@@ -229,7 +227,7 @@ class AddonManager(Mapping):"""if addon in self:del self[addon]- elif hasattr(addon, 'name') and addon.name in self:+ elif hasattr(addon, "name") and addon.name in self:del self[addon.name]else:try:@@ -250,14 +248,14 @@ class AddonManager(Mapping)::param path: Python or file path to an add-on:type path: ``str``"""- if isinstance(path, six.string_types):+ if isinstance(path, str):try:obj = load_module_or_object(path)except NameError:- raise NameError("Could not find add-on '%s'" % path)+ raise NameError(f"Could not find add-on '{path}'")else:obj = path- if hasattr(obj, '_addon'):+ if hasattr(obj, "_addon"):obj = AddonManager.get_addon(obj._addon)return obj@@ -284,7 +282,7 @@ class AddonManager(Mapping):and values correspond to their configuration:type addonsdict: ``dict``"""- for addonpath, addoncfg in six.iteritems(addonsdict):+ for addonpath, addoncfg in addonsdict.items():self.add(addonpath, addoncfg)def load_settings(self, settings):@@ -299,7 +297,7 @@ class AddonManager(Mapping):which to read the add-on configuration:type settings: :class:`~scrapy.settings.Settings`"""- paths = build_component_list(settings['ADDONS'])+ paths = build_component_list(settings["ADDONS"])addons = [self.get_addon(path) for path in paths]configs = [settings.getdict(addon.name.upper()) for addon in addons]for a, c in zip(addons, configs):@@ -322,20 +320,23 @@ class AddonManager(Mapping):add-on."""# Collect all active add-ons and the components they provide- ws = WorkingSet('')+ ws = WorkingSet("")def add_dist(project_name, version, **kwargs):- if project_name in ws.entry_keys.get('scrapy', []):- raise ImportError("Component {} provided by multiple add-ons"- "".format(project_name))+ if project_name in ws.entry_keys.get("scrapy", []):+ raise ImportError(+ f"Component {project_name} provided by multiple add-ons"+ )else:- dist = Distribution(project_name=project_name, version=version,- **kwargs)- ws.add(dist, entry='scrapy')+ dist = Distribution(+ project_name=project_name, version=version, **kwargs+ )+ ws.add(dist, entry="scrapy")+for name in self:ver = self[name].versionadd_dist(name, ver)- for provides_name in getattr(self[name], 'provides', []):+ for provides_name in getattr(self[name], "provides", []):add_dist(provides_name, ver)# Collect all required and modified components@@ -345,8 +346,9 @@ class AddonManager(Mapping):for entry in getattr(self[name], attribute_name, []):attrs[entry].append(name)return attrs- modified = compile_attribute_dict('modifies')- required = compile_attribute_dict('requires')++ modified = compile_attribute_dict("modifies")+ required = compile_attribute_dict("requires")req_or_mod = set(required.keys()).union(modified.keys())for reqstr in req_or_mod:@@ -355,15 +357,16 @@ class AddonManager(Mapping):# our own exception or is it helpful enough?if ws.find(req) is None:raise ImportError(- "Add-ons {} require or modify missing component {}"- "".format(required[reqstr]+modified[reqstr], reqstr)+ f"Add-ons {required[reqstr] + modified[reqstr]} require"+ f" or modify missing component {reqstr}")mod_and_req = set(required.keys()).intersection(modified.keys())for conflict in mod_and_req:- warnings.warn("Component '{}', required by add-ons {}, is modified "- "by add-ons {}".format(conflict, required[conflict],- modified[conflict]))+ warnings.warn(+ f"Component '{conflict}', required by add-ons {required[conflict]},"+ f" is modified by add-ons {modified[conflict]}"+ )def disable(self, addon):"""Disable an add-on, i.e. prevent its callbacks from being called.@@ -375,7 +378,7 @@ class AddonManager(Mapping)::type addon: ``str``"""if addon in self:- self.configs[addon]['_enabled'] = False+ self.configs[addon]["_enabled"] = Falseelse:self._disable_on_add.append(addon)@@ -389,23 +392,23 @@ class AddonManager(Mapping)::type addon: ``str``"""if addon in self:- self.configs[addon]['_enabled'] = True+ self.configs[addon]["_enabled"] = Trueelif addon in self._disable_on_add:self._disable_on_add.remove(addon)else:- raise ValueError("Add-ons need to be added before they can be "- "enabled")+ raise ValueError("Add-ons need to be added before they can be " "enabled")@propertydef disabled(self):"""Names of disabled add-ons"""- return ([a for a in self if not self.configs[a].get('_enabled', True)] +- self._disable_on_add)+ return [+ a for a in self if not self.configs[a].get("_enabled", True)+ ] + self._disable_on_add@propertydef enabled(self):"""Names of enabled add-ons"""- return [a for a in self if self.configs[a].get('_enabled', True)]+ return [a for a in self if self.configs[a].get("_enabled", True)]def _call_if_exists(self, obj, cbname, *args, **kwargs):if obj is None:@@ -418,9 +421,10 @@ class AddonManager(Mapping):cb(*args, **kwargs)def _call_addon(self, addonname, cbname, *args, **kwargs):- if self.configs[addonname].get('_enabled', True):- self._call_if_exists(self[addonname], cbname,- self.configs[addonname], *args, **kwargs)+ if self.configs[addonname].get("_enabled", True):+ self._call_if_exists(+ self[addonname], cbname, self.configs[addonname], *args, **kwargs+ )def update_addons(self):"""Call ``update_addons()`` of all held add-ons.@@ -432,7 +436,7 @@ class AddonManager(Mapping):while called_addons != set(self):for name in set(self).difference(called_addons):called_addons.add(name)- self._call_addon(name, 'update_addons', self)+ self._call_addon(name, "update_addons", self)def update_settings(self, settings):"""Call ``update_settings()`` of all held add-ons.@@ -442,7 +446,7 @@ class AddonManager(Mapping)::type settings: :class:`~scrapy.settings.Settings`"""for name in self:- self._call_addon(name, 'update_settings', settings)+ self._call_addon(name, "update_settings", settings)def check_configuration(self, crawler):"""Call ``check_configuration()`` of all held add-ons.@@ -451,7 +455,7 @@ class AddonManager(Mapping)::type crawler: :class:`~scrapy.crawler.Crawler`"""for name in self:- self._call_addon(name, 'check_configuration', crawler)+ self._call_addon(name, "check_configuration", crawler)-from scrapy.addons.builtins import *+from scrapy.addons.builtins import * # noqadiff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.pyindex 9babdeb6f..ea3afbf99 100644--- a/scrapy/addons/builtins.py+++ b/scrapy/addons/builtins.py@@ -1,23 +1,44 @@import scrapyfrom scrapy.addons import Addon-__all__ = ['make_builtin_addon',-- 'depth', 'httperror', 'offsite', 'referer', 'urllength',-- 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders',- 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression',- 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt',- 'stats', 'useragent',-- 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport',- 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump',- 'statsmailer', 'telnetconsole',- ]---def make_builtin_addon(addon_name, addon_default_config=None,- addon_version=None):+__all__ = [+ "make_builtin_addon",+ "depth",+ "httperror",+ "offsite",+ "referer",+ "urllength",+ "ajaxcrawl",+ "chunked",+ "cookies",+ "defaultheaders",+ "downloadtimeout",+ "httpauth",+ "httpcache",+ "httpcompression",+ "httpproxy",+ "metarefresh",+ "redirect",+ "retry",+ "robotstxt",+ "stats",+ "useragent",+ "autothrottle",+ "corestats",+ "closespider",+ "debugger",+ "feedexport",+ "logstats",+ "memdebug",+ "memusage",+ "spiderstate",+ "stacktracedump",+ "statsmailer",+ "telnetconsole",+]+++def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):class ThisAddon(Addon):name = addon_nameversion = addon_version or scrapy.__version__@@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None,# SPIDER MIDDLEWARES-depth = make_builtin_addon('depth')+depth = make_builtin_addon("depth")-httperror = make_builtin_addon('httperror')+httperror = make_builtin_addon("httperror")-offsite = make_builtin_addon('offsite')+offsite = make_builtin_addon("offsite")-referer = make_builtin_addon('referer')+referer = make_builtin_addon("referer")-urllength = make_builtin_addon('urllength')+urllength = make_builtin_addon("urllength")# DOWNLOADER MIDDLEWARES-ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True})+ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})-chunked = make_builtin_addon('chunked')+chunked = make_builtin_addon("chunked")++cookies = make_builtin_addon("cookies")++defaultheaders = make_builtin_addon("defaultheaders")-cookies = make_builtin_addon('cookies')-defaultheaders = make_builtin_addon('defaultheaders')# Assume every config entry is a headerdef defaultheaders_export_config(self, config, settings):conf = self.default_config or {}conf.update(config)- settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon')+ settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")++defaultheaders.export_config = defaultheaders_export_config-downloadtimeout = make_builtin_addon('downloadtimeout')-downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT',- 'download_timeout': 'DOWNLOAD_TIMEOUT'}+downloadtimeout = make_builtin_addon("downloadtimeout")+downloadtimeout.config_mapping = {+ "timeout": "DOWNLOAD_TIMEOUT",+ "download_timeout": "DOWNLOAD_TIMEOUT",+}-httpauth = make_builtin_addon('httpauth')+httpauth = make_builtin_addon("httpauth")-httpcache = make_builtin_addon('httpcache', {'enabled': True})+httpcache = make_builtin_addon("httpcache", {"enabled": True})-httpcompression = make_builtin_addon('httpcompression')-httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'}+httpcompression = make_builtin_addon("httpcompression")+httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}-httpproxy = make_builtin_addon('httpproxy')+httpproxy = make_builtin_addon("httpproxy")-metarefresh = make_builtin_addon('metarefresh')-metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'}+metarefresh = make_builtin_addon("metarefresh")+metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}-redirect = make_builtin_addon('redirect')+redirect = make_builtin_addon("redirect")-retry = make_builtin_addon('retry')+retry = make_builtin_addon("retry")-robotstxt = make_builtin_addon('robotstxt', {'obey': True})+robotstxt = make_builtin_addon("robotstxt", {"obey": True})-stats = make_builtin_addon('stats')+stats = make_builtin_addon("stats")-useragent = make_builtin_addon('useragent')-useragent.config_mapping = {'user_agent': 'USER_AGENT'}+useragent = make_builtin_addon("useragent")+useragent.config_mapping = {"user_agent": "USER_AGENT"}# ITEM PIPELINES@@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'}# EXTENSIONS-autothrottle = make_builtin_addon('autothrottle', {'enabled': True})+autothrottle = make_builtin_addon("autothrottle", {"enabled": True})-corestats = make_builtin_addon('corestats')+corestats = make_builtin_addon("corestats")-closespider = make_builtin_addon('closespider')+closespider = make_builtin_addon("closespider")-debugger = make_builtin_addon('debugger')+debugger = make_builtin_addon("debugger")-feedexport = make_builtin_addon('feedexport')-feedexport.settings_prefix = 'FEED'+feedexport = make_builtin_addon("feedexport")+feedexport.settings_prefix = "FEED"-logstats = make_builtin_addon('logstats')+logstats = make_builtin_addon("logstats")-memdebug = make_builtin_addon('memdebug', {'enabled': True})+memdebug = make_builtin_addon("memdebug", {"enabled": True})-memusage = make_builtin_addon('memusage', {'enabled': True})+memusage = make_builtin_addon("memusage", {"enabled": True})-spiderstate = make_builtin_addon('spiderstate')+spiderstate = make_builtin_addon("spiderstate")-stacktracedump = make_builtin_addon('stacktracedump')+stacktracedump = make_builtin_addon("stacktracedump")-statsmailer = make_builtin_addon('statsmailer')+statsmailer = make_builtin_addon("statsmailer")-telnetconsole = make_builtin_addon('telnetconsole')+telnetconsole = make_builtin_addon("telnetconsole")diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 4bf7e7e66..8577cce02 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -85,7 +85,7 @@ def load_module_or_object(path):return load_object(path)except (ValueError, NameError, ImportError):pass- raise NameError("Could not load '%s'" % path)+ raise NameError(f"Could not load '{path}'")def walk_modules(path):diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.pyindex a4e278fa5..741dd81cf 100644--- a/tests/test_addons/__init__.py+++ b/tests/test_addons/__init__.py@@ -1,145 +1,143 @@-from collections import OrderedDictimport itertools-import os.path-import six-from tests import mockimport unittestimport warnings+from collections import OrderedDict+from unittest import mockfrom pkg_resources import VersionConflict-import zope.interface+from zope.interface import directlyProvides+from zope.interface.exceptions import BrokenImplementation, MultipleInvalidfrom zope.interface.verify import verifyObject-from zope.interface.exceptions import BrokenImplementationfrom scrapy.addons import Addon, AddonManagerfrom scrapy.crawler import Crawlerfrom scrapy.interfaces import IAddonfrom scrapy.settings import BaseSettings-from . import addons-from . import addonmod+from . import addonmod, addonsclass AddonTest(unittest.TestCase):-def setUp(self):self.rawaddon = Addon()class AddonWithAttributes(Addon):- name = 'Test'- version = '1.0'+ name = "Test"+ version = "1.0"+self.testaddon = AddonWithAttributes()def test_interface(self):# Raw Addon should fail exactly b/c name and version are not given- self.assertFalse(hasattr(self.rawaddon, 'name'))- self.assertFalse(hasattr(self.rawaddon, 'version'))- self.assertRaises(BrokenImplementation, verifyObject, IAddon,- self.rawaddon)+ self.assertFalse(hasattr(self.rawaddon, "name"))+ self.assertFalse(hasattr(self.rawaddon, "version"))+ self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon)verifyObject(IAddon, self.testaddon)def test_export_component(self):- settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(),- 'DOWNLOAD_HANDLERS': BaseSettings()},- 'default')+ settings = BaseSettings(+ {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()},+ "default",+ )self.testaddon.component_type = Noneself.testaddon.export_component({}, settings)- self.assertEqual(len(settings['ITEM_PIPELINES']), 0)- self.testaddon.component_type = 'ITEM_PIPELINES'- self.testaddon.component = 'test.component'+ self.assertEqual(len(settings["ITEM_PIPELINES"]), 0)+ self.testaddon.component_type = "ITEM_PIPELINES"+ self.testaddon.component = "test.component"self.testaddon.export_component({}, settings)- six.assertCountEqual(self, settings['ITEM_PIPELINES'],- ['test.component'])- self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0)+ self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"])+ self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0)self.testaddon.component_order = 313self.testaddon.export_component({}, settings)- self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313)- self.testaddon.component_type = 'DOWNLOAD_HANDLERS'- self.testaddon.component_key = 'http'+ self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313)+ self.testaddon.component_type = "DOWNLOAD_HANDLERS"+ self.testaddon.component_key = "http"self.testaddon.export_component({}, settings)- self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'],- 'test.component')+ self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component")def test_export_basics(self):settings = BaseSettings()- self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True}+ self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True}self.testaddon.export_basics(settings)- self.assertEqual(settings['TESTKEY'], 313)- self.assertEqual(settings['OTHERKEY'], True)- self.assertEqual(settings.getpriority('TESTKEY'), 15)+ self.assertEqual(settings["TESTKEY"], 313)+ self.assertEqual(settings["OTHERKEY"], True)+ self.assertEqual(settings.getpriority("TESTKEY"), 15)def test_export_config(self):settings = BaseSettings()self.testaddon.settings_prefix = None- self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'}- self.testaddon.default_config = {'key': 55, 'defaultkey': 100}- self.testaddon.export_config({'key': 313, 'OTHERKEY': True,- 'mapped_KEY': 99}, settings)- self.assertEqual(settings['TEST_KEY'], 313)- self.assertEqual(settings['TEST_DEFAULTKEY'], 100)- self.assertEqual(settings['TEST_OTHERKEY'], True)- self.assertNotIn('MAPPED_key', settings)- self.assertNotIn('MAPPED_KEY', settings)- self.assertEqual(settings['MAPPING_WORKED'], 99)- self.assertEqual(settings.getpriority('TEST_KEY'), 15)-- self.testaddon.settings_prefix = 'PREF'- self.testaddon.export_config({'newkey': 99}, settings)- self.assertEqual(settings['PREF_NEWKEY'], 99)-- with mock.patch.object(settings, 'set') as mock_set:+ self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}+ self.testaddon.default_config = {"key": 55, "defaultkey": 100}+ self.testaddon.export_config(+ {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings+ )+ self.assertEqual(settings["TEST_KEY"], 313)+ self.assertEqual(settings["TEST_DEFAULTKEY"], 100)+ self.assertEqual(settings["TEST_OTHERKEY"], True)+ self.assertNotIn("MAPPED_key", settings)+ self.assertNotIn("MAPPED_KEY", settings)+ self.assertEqual(settings["MAPPING_WORKED"], 99)+ self.assertEqual(settings.getpriority("TEST_KEY"), 15)++ self.testaddon.settings_prefix = "PREF"+ self.testaddon.export_config({"newkey": 99}, settings)+ self.assertEqual(settings["PREF_NEWKEY"], 99)++ with mock.patch.object(settings, "set") as mock_set:self.testaddon.settings_prefix = False- self.testaddon.export_config({'thirdnewkey': 99}, settings)+ self.testaddon.export_config({"thirdnewkey": 99}, settings)self.assertEqual(mock_set.call_count, 0)def test_update_settings(self):settings = BaseSettings()- settings.set('TEST_KEY1', 'default', priority='default')- settings.set('TEST_KEY2', 'project', priority='project')+ settings.set("TEST_KEY1", "default", priority="default")+ settings.set("TEST_KEY2", "project", priority="project")self.testaddon.settings_prefix = None- self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'}- addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'}+ self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"}+ addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}self.testaddon.update_settings(addon_config, settings)- self.assertEqual(settings['OTHERTEST_KEY'], 'addon')- self.assertEqual(settings['TEST_KEY1'], 'addon')- self.assertEqual(settings['TEST_KEY2'], 'project')- self.assertEqual(settings['TEST_KEY3'], 'addon')+ self.assertEqual(settings["OTHERTEST_KEY"], "addon")+ self.assertEqual(settings["TEST_KEY1"], "addon")+ self.assertEqual(settings["TEST_KEY2"], "project")+ self.assertEqual(settings["TEST_KEY3"], "addon")class AddonManagerTest(unittest.TestCase):-def setUp(self):self.manager = AddonManager()def test_add(self):manager = AddonManager()- manager.add(addonmod, {'key': 'val1'})- manager.add('tests.test_addons.addons.GoodAddon')- six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon'])- self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)- six.assertCountEqual(self, manager.configs['AddonModule'], ['key'])- self.assertEqual(manager.configs['AddonModule']['key'], 'val1')+ manager.add(addonmod, {"key": "val1"})+ manager.add("tests.test_addons.addons.GoodAddon")+ self.assertCountEqual(manager, ["AddonModule", "GoodAddon"])+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)+ self.assertCountEqual(manager.configs["AddonModule"], ["key"])+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")self.assertRaises(ValueError, manager.add, addonmod)def test_add_dont_instantiate_providing_classes(self):class ProviderGoodAddon(addons.GoodAddon):pass- zope.interface.directlyProvides(ProviderGoodAddon, IAddon)++ directlyProvides(ProviderGoodAddon, IAddon)manager = AddonManager()manager.add(ProviderGoodAddon)- self.assertIs(manager['GoodAddon'], ProviderGoodAddon)+ self.assertIs(manager["GoodAddon"], ProviderGoodAddon)def test_add_verifies(self):- brokenaddon = self.manager.get_addon(- 'tests.test_addons.addons.BrokenAddon')- self.assertRaises(zope.interface.exceptions.BrokenImplementation,- self.manager.add,- brokenaddon)+ brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon")+ self.assertRaises(+ BrokenImplementation,+ self.manager.add,+ brokenaddon,+ )def test_add_adds_missing_interface_declaration(self):class GoodAddonWithoutDeclaration(object):- name = 'GoodAddonWithoutDeclaration'- version = '1.0'+ name = "GoodAddonWithoutDeclaration"+ version = "1.0"+self.manager.add(GoodAddonWithoutDeclaration)def test_remove(self):@@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase):def test_gets_removed(removearg):manager.add(addonmod)- self.assertIn('AddonModule', manager)+ self.assertIn("AddonModule", manager)manager.remove(removearg)- self.assertNotIn('AddonModule', manager)+ self.assertNotIn("AddonModule", manager)- test_gets_removed('AddonModule')+ test_gets_removed("AddonModule")test_gets_removed(addonmod)- test_gets_removed('tests.test_addons.addonmod')- self.assertRaises(KeyError, manager.remove, 'nonexistent')+ test_gets_removed("tests.test_addons.addonmod")+ self.assertRaises(KeyError, manager.remove, "nonexistent")self.assertRaises(KeyError, manager.remove, addons.GoodAddon())def test_get_addon(self):- goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon')+ goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon")self.assertIs(goodaddon, addons.GoodAddon)loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod")self.assertIs(loaded_addonmod, addonmod)- addonspath = os.path.join(os.path.dirname(__file__), 'addons.py')goodaddon = self.manager.get_addon("tests.test_addons.addons")self.assertIsInstance(goodaddon, addons.GoodAddon)- self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent')+ self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent")def test_get_addon_forward(self):class SomeCls(object):- _addon = 'tests.test_addons.addons.GoodAddon'+ _addon = "tests.test_addons.addons.GoodAddon"+self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon)def test_get_addon_nested(self):- x = addons.GoodAddon('outer')- x._addon = addons.GoodAddon('middle')- x._addon._addon = addons.GoodAddon('inner')+ x = addons.GoodAddon("outer")+ x._addon = addons.GoodAddon("middle")+ x._addon._addon = addons.GoodAddon("inner")self.assertIs(self.manager.get_addon(x), x._addon._addon)def test_load_dict_load_settings(self):def _test_load_method(func, *args, **kwargs):manager = AddonManager()getattr(manager, func)(*args, **kwargs)- six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule'])- self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)- six.assertCountEqual(self, manager.configs['GoodAddon'], ['key'])- self.assertEqual(manager.configs['GoodAddon']['key'], 'val2')- self.assertEqual(manager['AddonModule'], addonmod)- self.assertIn('key', manager.configs['AddonModule'])- self.assertEqual(manager.configs['AddonModule']['key'], 'val1')+ self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])+ self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)+ self.assertCountEqual(manager.configs["GoodAddon"], ["key"])+ self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")+ self.assertEqual(manager["AddonModule"], addonmod)+ self.assertIn("key", manager.configs["AddonModule"])+ self.assertEqual(manager.configs["AddonModule"]["key"], "val1")addonsdict = {- "tests.test_addons.addonmod": {'key': 'val1'},- 'tests.test_addons.addons.GoodAddon': {'key': 'val2'},- }- _test_load_method('load_dict', addonsdict)+ "tests.test_addons.addonmod": {"key": "val1"},+ "tests.test_addons.addons.GoodAddon": {"key": "val2"},+ }+ _test_load_method("load_dict", addonsdict)settings = BaseSettings()- settings.set('ADDONS', {"tests.test_addons.addonmod": 0,- 'tests.test_addons.addons.GoodAddon': 0})- settings.set('ADDONMODULE', {'key': 'val1'})- settings.set('GOODADDON', {'key': 'val2'})- _test_load_method('load_settings', settings)+ settings.set(+ "ADDONS",+ {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0},+ )+ settings.set("ADDONMODULE", {"key": "val1"})+ settings.set("GOODADDON", {"key": "val2"})+ _test_load_method("load_settings", settings)def test_load_dict_load_settings_order(self):def _test_load_method(expected_order, func, *args, **kwargs):@@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase):for ordered_addons in itertools.permutations(addonlist):expected_order = [a.name for a in ordered_addons]addonsdict = OrderedDict((a, {}) for a in ordered_addons)- _test_load_method(expected_order, 'load_dict', addonsdict)- settings = BaseSettings({- 'ADDONS': {a: i for i, a in enumerate(ordered_addons)}- })- _test_load_method(expected_order, 'load_settings', settings)+ _test_load_method(expected_order, "load_dict", addonsdict)+ settings = BaseSettings(+ {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}+ )+ _test_load_method(expected_order, "load_settings", settings)def test_enabled_disabled(self):manager = AddonManager()- manager.add(addons.GoodAddon('FirstAddon'))- manager.add(addons.GoodAddon('SecondAddon'))- self.assertEqual(set(manager.enabled),- set(('FirstAddon', 'SecondAddon')))+ manager.add(addons.GoodAddon("FirstAddon"))+ manager.add(addons.GoodAddon("SecondAddon"))+ self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))self.assertEqual(manager.disabled, [])- manager.disable('FirstAddon')- self.assertEqual(manager.enabled, ['SecondAddon'])- self.assertEqual(manager.disabled, ['FirstAddon'])- manager.enable('FirstAddon')- self.assertEqual(set(manager.enabled),- set(('FirstAddon', 'SecondAddon')))+ manager.disable("FirstAddon")+ self.assertEqual(manager.enabled, ["SecondAddon"])+ self.assertEqual(manager.disabled, ["FirstAddon"])+ manager.enable("FirstAddon")+ self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))self.assertEqual(manager.disabled, [])def test_enable_before_add(self):manager = AddonManager()- self.assertRaises(ValueError, manager.enable, 'FirstAddon')- manager.disable('FirstAddon')- manager.enable('FirstAddon')- manager.add(addons.GoodAddon('FirstAddon'))- self.assertIn('FirstAddon', manager.enabled)+ self.assertRaises(ValueError, manager.enable, "FirstAddon")+ manager.disable("FirstAddon")+ manager.enable("FirstAddon")+ manager.add(addons.GoodAddon("FirstAddon"))+ self.assertIn("FirstAddon", manager.enabled)def test_disable_before_add(self):manager = AddonManager()- manager.disable('FirstAddon')- manager.add(addons.GoodAddon('FirstAddon'))- self.assertEqual(manager.disabled, ['FirstAddon'])+ manager.disable("FirstAddon")+ manager.add(addons.GoodAddon("FirstAddon"))+ self.assertEqual(manager.disabled, ["FirstAddon"])def test_callbacks(self):- first_addon = addons.GoodAddon('FirstAddon')- second_addon = addons.GoodAddon('SecondAddon')+ first_addon = addons.GoodAddon("FirstAddon")+ second_addon = addons.GoodAddon("SecondAddon")manager = AddonManager()- manager.add(first_addon, {'test': 'first'})- manager.add(second_addon, {'test': 'second'})+ manager.add(first_addon, {"test": "first"})+ manager.add(second_addon, {"test": "second"})crawler = mock.create_autospec(Crawler)settings = BaseSettings()- with mock.patch.object(first_addon, 'update_addons') as ua_first, \- mock.patch.object(second_addon, 'update_addons') as ua_second, \- mock.patch.object(first_addon, 'update_settings') as us_first, \- mock.patch.object(second_addon, 'update_settings') as us_second, \- mock.patch.object(first_addon, 'check_configuration') as cc_first, \- mock.patch.object(second_addon, 'check_configuration') as cc_second:+ with mock.patch.object(+ first_addon, "update_addons"+ ) as ua_first, mock.patch.object(+ second_addon, "update_addons"+ ) as ua_second, mock.patch.object(+ first_addon, "update_settings"+ ) as us_first, mock.patch.object(+ second_addon, "update_settings"+ ) as us_second, mock.patch.object(+ first_addon, "check_configuration"+ ) as cc_first, mock.patch.object(+ second_addon, "check_configuration"+ ) as cc_second:manager.update_addons()- ua_first.assert_called_once_with(manager.configs['FirstAddon'],- manager)- ua_second.assert_called_once_with(manager.configs['SecondAddon'],- manager)+ ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)+ ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager)manager.update_settings(settings)- us_first.assert_called_once_with(manager.configs['FirstAddon'],- settings)- us_second.assert_called_once_with(manager.configs['SecondAddon'],- settings)+ us_first.assert_called_once_with(manager.configs["FirstAddon"], settings)+ us_second.assert_called_once_with(manager.configs["SecondAddon"], settings)manager.check_configuration(crawler)- cc_first.assert_called_once_with(manager.configs['FirstAddon'],- crawler)- cc_second.assert_called_once_with(manager.configs['SecondAddon'],- crawler)+ cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler)+ cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler)self.assertEqual(ua_first.call_count, 1)self.assertEqual(ua_second.call_count, 1)self.assertEqual(us_first.call_count, 1)@@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase):us_first.reset_mock()us_second.reset_mock()- manager.disable('FirstAddon')+ manager.disable("FirstAddon")manager.update_settings(settings)self.assertEqual(us_first.call_count, 0)- manager.enable('FirstAddon')+ manager.enable("FirstAddon")manager.update_settings(settings)self.assertEqual(us_first.call_count, 1)self.assertEqual(us_second.call_count, 2)@@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase):# This will become relevant when we let spiders implement the add-on# interface and should be replaced with a test where# AddonManager.spidercls = None then.- manager._call_if_exists(None, 'irrelevant')+ manager._call_if_exists(None, "irrelevant")def test_update_addons_last_minute_add(self):class AddedAddon(addons.GoodAddon):- name = 'AddedAddon'+ name = "AddedAddon"class FirstAddon(addons.GoodAddon):- name = 'FirstAddon'+ name = "FirstAddon"def update_addons(self, config, addons):addons.add(AddedAddon())manager = AddonManager()first_addon = FirstAddon()- with mock.patch.object(first_addon, 'update_addons',- wraps=first_addon.update_addons) as ua_first, \- mock.patch.object(AddedAddon, 'update_addons') as ua_added:- manager.add(first_addon, {'non-empty': 'dict'})+ with mock.patch.object(+ first_addon, "update_addons", wraps=first_addon.update_addons+ ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added:+ manager.add(first_addon, {"non-empty": "dict"})manager.update_addons()- six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon'])- ua_first.assert_called_once_with(manager.configs['FirstAddon'],- manager)- ua_added.assert_called_once_with(manager.configs['AddedAddon'],- manager)+ self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"])+ ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)+ ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager)def test_check_dependency_clashes_attributes(self):provides = addons.GoodAddon("ProvidesAddon")- provides.provides = ('test', )+ provides.provides = ("test",)provides2 = addons.GoodAddon("ProvidesAddon2")- provides2.provides = ('test', )+ provides2.provides = ("test",)requires = addons.GoodAddon("RequiresAddon")- requires.requires = ('test', )+ requires.requires = ("test",)requires_name = addons.GoodAddon("RequiresNameAddon")- requires_name.requires = ('ProvidesAddon', )+ requires_name.requires = ("ProvidesAddon",)requires_newer = addons.GoodAddon("RequiresNewerAddon")- requires_newer.requires = ('test>=2.0', )+ requires_newer.requires = ("test>=2.0",)modifies = addons.GoodAddon("ModifiesAddon")- modifies.modifies = ('test', )+ modifies.modifies = ("test",)def check_with(*addons):manager = AddonManager()diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.pyindex 8ecf4b81d..c59f6737f 100644--- a/tests/test_addons/addonmod.py+++ b/tests/test_addons/addonmod.py@@ -9,8 +9,10 @@ FROM = "test_addons.addonmod"name = "AddonModule"version = "1.0"+def update_settings(config, settings):pass+def check_configuration(config, crawler):passdiff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.pyindex f3442b192..4adb9fe8f 100644--- a/tests/test_addons/addons.py+++ b/tests/test_addons/addons.py@@ -1,18 +1,16 @@import zope.interface-from scrapy.addons import Addonfrom scrapy.interfaces import IAddonclass Addon(object):- FROM = 'test_addons.addons'+ FROM = "test_addons.addons"@zope.interface.declarations.implementer(IAddon)class GoodAddon(object):-- name = 'GoodAddon'- version = '1.0'+ name = "GoodAddon"+ version = "1.0"def __init__(self, name=None, version=None):if name is not None:@@ -32,8 +30,7 @@ class GoodAddon(object):@zope.interface.declarations.implementer(IAddon)class BrokenAddon(object):-- name = 'BrokenAddon'+ name = "BrokenAddon"# No versiondiff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.pyindex c89876950..1050cbbed 100644--- a/tests/test_addons/test_builtins.py+++ b/tests/test_addons/test_builtins.py@@ -7,19 +7,17 @@ from scrapy.settings import Settingsclass BuiltinAddonsTest(unittest.TestCase):-def test_make_builtin_addon(self):- httpcache = make_builtin_addon('httpcache', {'enabled': True})- self.assertEqual(httpcache.name, 'httpcache')- self.assertEqual(httpcache.default_config, {'enabled': True})+ httpcache = make_builtin_addon("httpcache", {"enabled": True})+ self.assertEqual(httpcache.name, "httpcache")+ self.assertEqual(httpcache.default_config, {"enabled": True})self.assertEqual(httpcache.version, scrapy.__version__)- httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9')- self.assertEqual(httpcache.version, '99.9')+ httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")+ self.assertEqual(httpcache.version, "99.9")def test_defaultheaders_export_config(self):settings = Settings()dh = scrapy.addons.defaultheaders()- dh.export_config({'X-Test-Header': 'val'}, settings)- self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS'])- self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'],- 'val')+ dh.export_config({"X-Test-Header": "val"}, settings)+ self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])+ self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex e353f80cb..1e97863b0 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -419,7 +419,7 @@ with multiples linesaddonmgr = AddonManager()addonmgr.add(FailedCheckAddon())- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)crawler.addons = addonmgr# Doesn't work in 'precise' test environment:# with self.assertRaises(ValueError):diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex 7ddf952f7..fd57d846e 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase):def test_instances_from_settings(self):settings = Settings()- myM3 = M3()class InstanceTestMiddlewareManager(MiddlewareManager):@classmethoddef _get_mwlist_from_settings(cls, settings):- return ["tests.test_middleware.M1", M2, myM3]+ return ["tests.test_middleware.M1", M2]mwman = InstanceTestMiddlewareManager.from_settings(settings)self.assertIsInstance(mwman.middlewares[0], M1)self.assertIsInstance(mwman.middlewares[1], M2)- self.assertIs(mwman.middlewares[2], myM3)