Scrapy, a fast high-level web crawling & scraping framework for Python.
Change super syntax (#4707)
Marc Hernández336f19f5cc6e
diff --git a/extras/qpsclient.py b/extras/qpsclient.pyindex 7554f7eec..fe1f96cbb 100644--- a/extras/qpsclient.py+++ b/extras/qpsclient.py@@ -27,7 +27,7 @@ class QPSSpider(Spider):slots = 1def __init__(self, *a, **kw):- super(QPSSpider, self).__init__(*a, **kw)+ super().__init__(*a, **kw)if self.qps is not None:self.qps = float(self.qps)self.download_delay = 1 / self.qpsdiff --git a/scrapy/commands/view.py b/scrapy/commands/view.pyindex 908bee966..c8f873334 100644--- a/scrapy/commands/view.py+++ b/scrapy/commands/view.py@@ -11,7 +11,7 @@ class Command(fetch.Command):return "Fetch a URL using the Scrapy downloader and show its contents in a browser"def add_options(self, parser):- super(Command, self).add_options(parser)+ super().add_options(parser)parser.remove_option("--headers")def _print_response(self, response, opts):diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.pyindex 34f0d36d4..cfdcc7c25 100644--- a/scrapy/contracts/default.py+++ b/scrapy/contracts/default.py@@ -56,7 +56,7 @@ class ReturnsContract(Contract):}def __init__(self, *args, **kwargs):- super(ReturnsContract, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)if len(self.args) not in [1, 2, 3]:raise ValueError(diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 452242d47..8a7d656a1 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):"""def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self._ssl_method = methodself.tls_verbose_logging = tls_verbose_loggingif tls_ciphers:@@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)## * getattr() for `_ssl_method` attribute for context factories- # not calling super(..., self).__init__+ # not calling super().__init__return CertificateOptions(verify=False,method=getattr(self, 'method', getattr(self, '_ssl_method', None)),diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 22c9ac520..fb04d1fb7 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):proxyHost, proxyPort, self._proxyAuthHeader = proxyConf- super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)+ super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)self._tunnelReadyDeferred = defer.Deferred()self._tunneledHost = hostself._tunneledPort = port@@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):def connect(self, protocolFactory):self._protocolFactory = protocolFactory- connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)+ connectDeferred = super().connect(protocolFactory)connectDeferred.addCallback(self.requestTunnel)connectDeferred.addErrback(self.connectFailed)return self._tunnelReadyDeferred@@ -215,7 +215,7 @@ class TunnelingAgent(Agent):def __init__(self, reactor, proxyConf, contextFactory=None,connectTimeout=None, bindAddress=None, pool=None):- super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)+ super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)self._proxyConf = proxyConfself._contextFactory = contextFactory@@ -235,7 +235,7 @@ class TunnelingAgent(Agent):# otherwise, same remote host connection request could reuse# a cached tunneled connection to a different proxykey = key + self._proxyConf- return super(TunnelingAgent, self)._requestWithEndpoint(+ return super()._requestWithEndpoint(key=key,endpoint=endpoint,method=method,@@ -249,7 +249,7 @@ class TunnelingAgent(Agent):class ScrapyProxyAgent(Agent):def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):- super(ScrapyProxyAgent, self).__init__(+ super().__init__(reactor=reactor,connectTimeout=connectTimeout,bindAddress=bindAddress,diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex e43a3c83e..d9f3750d5 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):"""def __init__(self, hostname, ctx, verbose_logging=False):- super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)+ super().__init__(hostname, ctx)self.verbose_logging = verbose_loggingdef _identityVerifyingInfoCallback(self, connection, where, ret):diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 35264a92b..5a99b96be 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))def _add_middleware(self, mw):- super(SpiderMiddlewareManager, self)._add_middleware(mw)+ super()._add_middleware(mw)if hasattr(mw, 'process_spider_input'):self.methods['process_spider_input'].append(mw.process_spider_input)if hasattr(mw, 'process_start_requests'):diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 6f43771e2..48f19424c 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):"""def __init__(self, settings=None, install_root_handler=True):- super(CrawlerProcess, self).__init__(settings)+ super().__init__(settings)install_shutdown_handlers(self._signal_shutdown)configure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 366d60dcb..4053fecc5 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):enabled_setting = 'METAREFRESH_ENABLED'def __init__(self, settings):- super(MetaRefreshMiddleware, self).__init__(settings)+ super().__init__(settings)self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 45f152321..0c410f035 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -37,7 +37,7 @@ class CloseSpider(Exception):"""Raise this from callbacks to request the spider to be closed"""def __init__(self, reason='cancelled'):- super(CloseSpider, self).__init__()+ super().__init__()self.reason = reason@@ -74,7 +74,7 @@ class UsageError(Exception):def __init__(self, *a, **kw):self.print_help = kw.pop('print_help', True)- super(UsageError, self).__init__(*a, **kw)+ super().__init__(*a, **kw)class ScrapyDeprecationWarning(Warning):diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 0aba1c904..95518b3ac 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):def _configure(self, options, dont_fail=False):self.binary = options.pop('binary', True)- super(PythonItemExporter, self)._configure(options, dont_fail)+ super()._configure(options, dont_fail)if self.binary:warnings.warn("PythonItemExporter will drop support for binary export in the future",diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex dcaaeddfa..6bf9e5346 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -8,7 +8,7 @@ class Headers(CaselessDict):def __init__(self, seq=None, encoding='utf-8'):self.encoding = encoding- super(Headers, self).__init__(seq)+ super().__init__(seq)def normkey(self, key):"""Normalize key to bytes"""@@ -37,19 +37,19 @@ class Headers(CaselessDict):def __getitem__(self, key):try:- return super(Headers, self).__getitem__(key)[-1]+ return super().__getitem__(key)[-1]except IndexError:return Nonedef get(self, key, def_val=None):try:- return super(Headers, self).get(key, def_val)[-1]+ return super().get(key, def_val)[-1]except IndexError:return Nonedef getlist(self, key, def_val=None):try:- return super(Headers, self).__getitem__(key)+ return super().__getitem__(key)except KeyError:if def_val is not None:return self.normvalue(def_val)diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex a260798ac..59af81321 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -24,7 +24,7 @@ class FormRequest(Request):if formdata and kwargs.get('method') is None:kwargs['method'] = 'POST'- super(FormRequest, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)if formdata:items = formdata.items() if isinstance(formdata, dict) else formdatadiff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.pyindex f08b25280..eae3f9f6b 100644--- a/scrapy/http/request/json_request.py+++ b/scrapy/http/request/json_request.py@@ -32,7 +32,7 @@ class JsonRequest(Request):if 'method' not in kwargs:kwargs['method'] = 'POST'- super(JsonRequest, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.headers.setdefault('Content-Type', 'application/json')self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')@@ -47,7 +47,7 @@ class JsonRequest(Request):elif not body_passed and data_passed:kwargs['body'] = self._dumps(data)- return super(JsonRequest, self).replace(*args, **kwargs)+ return super().replace(*args, **kwargs)def _dumps(self, data):"""Convert to JSON """diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.pyindex 811d3ad6b..c70912e49 100644--- a/scrapy/http/request/rpc.py+++ b/scrapy/http/request/rpc.py@@ -31,5 +31,5 @@ class XmlRpcRequest(Request):if encoding is not None:kwargs['encoding'] = encoding- super(XmlRpcRequest, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.headers.setdefault('Content-Type', 'text/xml')diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 0f300c8da..a7bb34d48 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -35,13 +35,13 @@ class TextResponse(Response):self._cached_benc = Noneself._cached_ubody = Noneself._cached_selector = None- super(TextResponse, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)def _set_url(self, url):if isinstance(url, str):self._url = to_unicode(url, self.encoding)else:- super(TextResponse, self)._set_url(url)+ super()._set_url(url)def _set_body(self, body):self._body = b'' # used by encoding detection@@ -51,7 +51,7 @@ class TextResponse(Response):type(self).__name__)self._body = body.encode(self._encoding)else:- super(TextResponse, self)._set_body(body)+ super()._set_body(body)def replace(self, *args, **kwargs):kwargs.setdefault('encoding', self.encoding)@@ -166,7 +166,7 @@ class TextResponse(Response):elif isinstance(url, parsel.SelectorList):raise ValueError("SelectorList is not supported")encoding = self.encoding if encoding is None else encoding- return super(TextResponse, self).follow(+ return super().follow(url=url,callback=callback,method=method,@@ -226,7 +226,7 @@ class TextResponse(Response):for sel in selectors:with suppress(_InvalidSelector):urls.append(_url_from_selector(sel))- return super(TextResponse, self).follow_all(+ return super().follow_all(urls=urls,callback=callback,method=method,diff --git a/scrapy/item.py b/scrapy/item.pyindex 4ab83d1a0..c262a153c 100644--- a/scrapy/item.py+++ b/scrapy/item.py@@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',ScrapyDeprecationWarning, stacklevel=2)- return super(BaseItem, cls).__new__(cls, *args, **kwargs)+ return super().__new__(cls, *args, **kwargs)class Field(dict):@@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):def __new__(mcs, class_name, bases, attrs):classcell = attrs.pop('__classcell__', None)new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))- _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)+ _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)fields = getattr(_class, 'fields', {})new_attrs = {}@@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):new_attrs['_class'] = _classif classcell is not None:new_attrs['__classcell__'] = classcell- return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)+ return super().__new__(mcs, class_name, bases, new_attrs)class DictItem(MutableMapping, BaseItem):@@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):if issubclass(cls, DictItem) and not issubclass(cls, Item):warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',ScrapyDeprecationWarning, stacklevel=2)- return super(DictItem, cls).__new__(cls, *args, **kwargs)+ return super().__new__(cls, *args, **kwargs)def __init__(self, *args, **kwargs):self._values = {}@@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):def __setattr__(self, name, value):if not name.startswith('_'):raise AttributeError("Use item[%r] = %r to set field value" % (name, value))- super(DictItem, self).__setattr__(name, value)+ super().__setattr__(name, value)def __len__(self):return len(self._values)diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex 984a5c4e1..08a6ca1e8 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -65,7 +65,7 @@ class FilteringLinkExtractor:warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ''please use scrapy.linkextractors.LinkExtractor instead',ScrapyDeprecationWarning, stacklevel=2)- return super(FilteringLinkExtractor, cls).__new__(cls)+ return super().__new__(cls)def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 8b9f961ee..e941c4321 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):strip=strip,canonicalized=canonicalize)- super(LxmlLinkExtractor, self).__init__(+ super().__init__(link_extractor=lx,allow=allow,deny=deny,diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 487382a38..6bc5d46eb 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD)- super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)+ super().__init__(download_func=download_func, settings=settings)@classmethoddef from_settings(cls, settings):diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 46f2bfb58..e2dd70215 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):DEFAULT_IMAGES_RESULT_FIELD = 'images'def __init__(self, store_uri, download_func=None, settings=None):- super(ImagesPipeline, self).__init__(store_uri, settings=settings,- download_func=download_func)+ super().__init__(store_uri, settings=settings, download_func=download_func)if isinstance(settings, dict) or settings is None:settings = Settings(settings)diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex f69894b1e..f191deac6 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):"""def __init__(self, reactor, cache_size, timeout):- super(CachingThreadedResolver, self).__init__(reactor)+ super().__init__(reactor)dnscache.limit = cache_sizeself.timeout = timeout@@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):# so the input argument above is simply overridden# to enforce Scrapy's DNS_TIMEOUT setting's valuetimeout = (self.timeout,)- d = super(CachingThreadedResolver, self).getHostByName(name, timeout)+ d = super().getHostByName(name, timeout)if dnscache.limit:d.addCallback(self._cache_result, name)return d@@ -80,16 +80,16 @@ class CachingHostnameResolver:class CachingResolutionReceiver(resolutionReceiver):def resolutionBegan(self, resolution):- super(CachingResolutionReceiver, self).resolutionBegan(resolution)+ super().resolutionBegan(resolution)self.resolution = resolutionself.resolved = Falsedef addressResolved(self, address):- super(CachingResolutionReceiver, self).addressResolved(address)+ super().addressResolved(address)self.resolved = Truedef resolutionComplete(self):- super(CachingResolutionReceiver, self).resolutionComplete()+ super().resolutionComplete()if self.resolved:dnscache[hostName] = self.resolutiondiff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex 85a9bb526..f12c61081 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):kwargs.setdefault('base_url', response.url)self.response = response- super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)+ super().__init__(text=text, type=st, root=root, **kwargs)diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex ff8317cd1..b8ae32d7c 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -439,7 +439,7 @@ class Settings(BaseSettings):# Do not pass kwarg values here. We don't want to promote user-defined# dicts, and we want to update, not replace, default dicts with the# values given by the user- super(Settings, self).__init__()+ super().__init__()self.setmodule(default_settings, 'default')# Promote default dictionaries to BaseSettings instances for per-key# prioritiesdiff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 375042340..db9d0f2ae 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):def __init__(self, response, *args, **kwargs):self.response = response- super(HttpError, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)class HttpErrorMiddleware:diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex cb7260892..c9fbce08d 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -75,7 +75,7 @@ class CrawlSpider(Spider):rules = ()def __init__(self, *a, **kw):- super(CrawlSpider, self).__init__(*a, **kw)+ super().__init__(*a, **kw)self._compile_rules()def _parse(self, response, **kwargs):@@ -145,6 +145,6 @@ class CrawlSpider(Spider):@classmethoddef from_crawler(cls, crawler, *args, **kwargs):- spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)+ spider = super().from_crawler(crawler, *args, **kwargs)spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)return spiderdiff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.pyindex fd41133ea..fe8c94e78 100644--- a/scrapy/spiders/init.py+++ b/scrapy/spiders/init.py@@ -6,7 +6,7 @@ class InitSpider(Spider):"""Base Spider with initialization facilities"""def start_requests(self):- self._postinit_reqs = super(InitSpider, self).start_requests()+ self._postinit_reqs = super().start_requests()return iterate_spider_output(self.init_request())def initialized(self, response=None):diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex c5360bfa7..1f72e76b7 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -18,7 +18,7 @@ class SitemapSpider(Spider):sitemap_alternate_links = Falsedef __init__(self, *a, **kw):- super(SitemapSpider, self).__init__(*a, **kw)+ super().__init__(*a, **kw)self._cbs = []for r, c in self.sitemap_rules:if isinstance(c, str):diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex c7ad4d53d..77ffda6f7 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -20,7 +20,7 @@ def _with_mkdir(queue_class):if not os.path.exists(dirname):os.makedirs(dirname, exist_ok=True)- super(DirectoriesCreated, self).__init__(path, *args, **kwargs)+ super().__init__(path, *args, **kwargs)return DirectoriesCreated@@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):def push(self, obj):s = serialize(obj)- super(SerializableQueue, self).push(s)+ super().push(s)def pop(self):- s = super(SerializableQueue, self).pop()+ s = super().pop()if s:return deserialize(s)@@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):def __init__(self, crawler, key):self.spider = crawler.spider- super(ScrapyRequestQueue, self).__init__(key)+ super().__init__(key)@classmethoddef from_crawler(cls, crawler, key, *args, **kwargs):@@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):def push(self, request):request = request_to_dict(request, self.spider)- return super(ScrapyRequestQueue, self).push(request)+ return super().push(request)def pop(self):- request = super(ScrapyRequestQueue, self).pop()+ request = super().pop()if not request:return Nonediff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.pyindex 579c60180..ba7d1a6bf 100644--- a/scrapy/statscollectors.py+++ b/scrapy/statscollectors.py@@ -54,7 +54,7 @@ class StatsCollector:class MemoryStatsCollector(StatsCollector):def __init__(self, crawler):- super(MemoryStatsCollector, self).__init__(crawler)+ super().__init__(crawler)self.spider_stats = {}def _persist_stats(self, stats, spider):diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 2a92d0588..e31284a7f 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -15,7 +15,7 @@ class CaselessDict(dict):__slots__ = ()def __init__(self, seq=None):- super(CaselessDict, self).__init__()+ super().__init__()if seq:self.update(seq)@@ -53,7 +53,7 @@ class CaselessDict(dict):def update(self, seq):seq = seq.items() if isinstance(seq, Mapping) else seqiseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)- super(CaselessDict, self).update(iseq)+ super().update(iseq)@classmethoddef fromkeys(cls, keys, value=None):@@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):"""def __init__(self, limit=None):- super(LocalCache, self).__init__()+ super().__init__()self.limit = limitdef __setitem__(self, key, value):if self.limit:while len(self) >= self.limit:self.popitem(last=False)- super(LocalCache, self).__setitem__(key, value)+ super().__setitem__(key, value)class LocalWeakReferencedCache(weakref.WeakKeyDictionary):@@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):"""def __init__(self, limit=None):- super(LocalWeakReferencedCache, self).__init__()+ super().__init__()self.data = LocalCache(limit=limit)def __setitem__(self, key, value):try:- super(LocalWeakReferencedCache, self).__setitem__(key, value)+ super().__setitem__(key, value)except TypeError:pass # key is not weak-referenceable, skip cachingdef __getitem__(self, key):try:- return super(LocalWeakReferencedCache, self).__getitem__(key)+ return super().__getitem__(key)except (TypeError, KeyError):return None # key is either not weak-referenceable or not cacheddiff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 3dbea5fee..3c8e3c8b5 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -57,7 +57,7 @@ def create_deprecated_class(warned_on_subclass = Falsedef __new__(metacls, name, bases, clsdict_):- cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)+ cls = super().__new__(metacls, name, bases, clsdict_)if metacls.deprecated_class is None:metacls.deprecated_class = clsreturn cls@@ -73,7 +73,7 @@ def create_deprecated_class(if warn_once:msg += ' (warning only on first subclass, there may be others)'warnings.warn(msg, warn_category, stacklevel=2)- super(DeprecatedClass, cls).__init__(name, bases, clsdict_)+ super().__init__(name, bases, clsdict_)# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks@@ -88,7 +88,7 @@ def create_deprecated_class(# is the deprecated class itself - subclasses of the# deprecated class should not use custom `__subclasscheck__`# method.- return super(DeprecatedClass, cls).__subclasscheck__(sub)+ return super().__subclasscheck__(sub)if not inspect.isclass(sub):raise TypeError("issubclass() arg 1 must be a class")@@ -102,7 +102,7 @@ def create_deprecated_class(msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),new=_clspath(new_class, new_class_path))warnings.warn(msg, warn_category, stacklevel=2)- return super(DeprecatedClass, cls).__call__(*args, **kwargs)+ return super().__call__(*args, **kwargs)deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 51d276097..1d6a2c39d 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):"""Record log levels count into a crawler stats"""def __init__(self, crawler, *args, **kwargs):- super(LogCounterHandler, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.crawler = crawlerdef emit(self, record):diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.pyindex dc9604578..cc3263602 100644--- a/scrapy/utils/serialize.py+++ b/scrapy/utils/serialize.py@@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):elif isinstance(o, Response):return "<%s %s %s>" % (type(o).__name__, o.status, o.url)else:- return super(ScrapyJSONEncoder, self).default(o)+ return super().default(o)class ScrapyJSONDecoder(json.JSONDecoder):diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex 66930ad2c..397e54703 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -7,12 +7,12 @@ class SiteTest:def setUp(self):from twisted.internet import reactor- super(SiteTest, self).setUp()+ super().setUp()self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")self.baseurl = "http://localhost:%d/" % self.site.getHost().portdef tearDown(self):- super(SiteTest, self).tearDown()+ super().tearDown()self.site.stopListening()def url(self, path):diff --git a/tests/spiders.py b/tests/spiders.pyindex 3eb681819..63bd726fb 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queueclass MockServerSpider(Spider):def __init__(self, mockserver=None, *args, **kwargs):- super(MockServerSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.mockserver = mockserver@@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):name = 'meta'def __init__(self, *args, **kwargs):- super(MetaSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.meta = {}def closed(self, reason):@@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):link_extractor = LinkExtractor()def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):- super(FollowAllSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.urls_visited = []self.times = []qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}@@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):name = 'delay'def __init__(self, n=1, b=0, *args, **kwargs):- super(DelaySpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.n = nself.b = bself.t1 = self.t2 = self.t2_err = 0@@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):name = 'simple'def __init__(self, url="http://localhost:8998", *args, **kwargs):- super(SimpleSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.start_urls = [url]def parse(self, response):@@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):name = 'item'def parse(self, response):- for request in super(ItemSpider, self).parse(response):+ for request in super().parse(response):yield requestyield Item()yield {}@@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):raise self.exception_cls('Expected exception')def parse(self, response):- for request in super(ErrorSpider, self).parse(response):+ for request in super().parse(response):yield requestself.raise_exception()@@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):fail_yielding = Falsedef __init__(self, *a, **kw):- super(BrokenStartRequestsSpider, self).__init__(*a, **kw)+ super().__init__(*a, **kw)self.seedsseen = []def start_requests(self):@@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):def parse(self, response):self.seedsseen.append(response.meta.get('seed'))- for req in super(BrokenStartRequestsSpider, self).parse(response):+ for req in super().parse(response):yield req@@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):yield Request(url, dont_filter=self.dont_filter)def __init__(self, url="http://localhost:8998", *args, **kwargs):- super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.visited = 0def parse(self, response):diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 5754a5478..e115f420f 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):command = 'parse'def setUp(self):- super(ParseCommandTest, self).setUp()+ super().setUp()self.spider_name = 'parse_spider'fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))with open(fname, 'w') as f:diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 42091ab00..8938156fc 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):class StartprojectTemplatesTest(ProjectTest):def setUp(self):- super(StartprojectTemplatesTest, self).setUp()+ super().setUp()self.tmpl = join(self.temp_path, 'templates')self.tmpl_proj = join(self.tmpl, 'project')@@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):class CommandTest(ProjectTest):def setUp(self):- super(CommandTest, self).setUp()+ super().setUp()self.call('startproject', self.project_name)self.cwd = join(self.temp_path, self.project_name)self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_namediff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex 99120b128..2e7e3ccc4 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):name = 'test_same_url'def __init__(self, *args, **kwargs):- super(TestSameUrlSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.visited = 0def start_requests(s):diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 57d4cdd6b..13063d106 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""def setUp(self):- super(Https11InvalidDNSId, self).setUp()+ super().setUp()self.host = '127.0.0.1'@@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ''subject "/C=IE/O=Scrapy/CN=127.0.0.1"')- super(Https11InvalidDNSPattern, self).setUp()+ super().setUp()class Https11CustomCiphers(unittest.TestCase):diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 9b77c97a8..299fb0eb8 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):def _get_settings(self, **new_settings):new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)- return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)+ return super()._get_settings(**new_settings)def test_custom_dbm_module_loaded(self):# make sure our dbm module has been loaded@@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):def _get_settings(self, **new_settings):new_settings.setdefault('HTTPCACHE_GZIP', True)- return super(FilesystemStorageTest, self)._get_settings(**new_settings)+ return super()._get_settings(**new_settings)class DummyPolicyTest(_BaseTest):diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex f9936baba..858138f81 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):skip = "Rerp parser is not installed"def setUp(self):- super(RobotsTxtMiddlewareWithRerpTest, self).setUp()+ super().setUp()self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')@@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):skip = "Reppy parser is not installed"def setUp(self):- super(RobotsTxtMiddlewareWithReppyTest, self).setUp()+ super().setUp()self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 660c99ce1..6c25a0064 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase):if name == 'age':return str(int(value) + 1)else:- return super(CustomItemExporter, self).serialize_field(field, name, value)+ return super().serialize_field(field, name, value)i = self.item_class(name='John', age='22')a = ItemAdapter(i)diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex f5cf4e798..0a303dbe2 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):def setUp(self):warnings.simplefilter("always")- super(JsonRequestTest, self).setUp()+ super().setUp()def test_data(self):r1 = self.request_class(url="http://www.example.com/")@@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):def tearDown(self):warnings.resetwarnings()- super(JsonRequestTest, self).tearDown()+ super().tearDown()if __name__ == "__main__":diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 56d017de6..f831ef5dc 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):response_class = TextResponsedef test_replace(self):- super(TextResponseTest, self).test_replace()+ super().test_replace()r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")r2 = r1.replace(url="http://www.example.com/other")r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")diff --git a/tests/test_item.py b/tests/test_item.pyindex 0ce78f8c0..66fa761f0 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):# requirement. When not done properly raises an error:# TypeError: __class__ set to <class '__main__.MyItem'># defining 'MyItem' as <class '__main__.MyItem'>- super(MyItem, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)class DictItemTest(unittest.TestCase):diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex a0bafa5e5..6f133d77a 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):])def test_restrict_xpaths_with_html_entities(self):- super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()+ super().test_restrict_xpaths_with_html_entities()def test_filteringlinkextractor_deprecation_warning(self):"""Make sure the FilteringLinkExtractor deprecation warning is notdiff --git a/tests/test_loader.py b/tests/test_loader.pyindex 2ed6f365f..b0bc82f4e 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):temp = Field()def __init__(self, *args, **kwargs):- super(TempItem, self).__init__(self, *args, **kwargs)+ super().__init__(self, *args, **kwargs)self.setdefault('temp', 0.3)class TempLoader(ItemLoader):diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex eb14de14f..624dd9ab8 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):class TempDict(dict):def __init__(self, *args, **kwargs):- super(TempDict, self).__init__(self, *args, **kwargs)+ super().__init__(self, *args, **kwargs)self.setdefault('temp', 0.3)class TempLoader(ItemLoader):diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex b771e7d79..41ff3651d 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase):class LogFormatterSubclass(LogFormatter):def crawled(self, request, response, spider):- kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)+ kwargs = super().crawled(request, response, spider)CRAWLEDMSG = ("Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s")diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex 3364d2258..b2b75ef20 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]def _add_middleware(self, mw):- super(TestMiddlewareManager, self)._add_middleware(mw)+ super()._add_middleware(mw)if hasattr(mw, 'process'):self.methods['process'].append(mw.process)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 19ff00350..4f130c0c9 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):class MockedMediaPipeline(MediaPipeline):def __init__(self, *args, **kwargs):- super(MockedMediaPipeline, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self._mockcalled = []def download(self, request, info):self._mockcalled.append('download')- return super(MockedMediaPipeline, self).download(request, info)+ return super().download(request, info)def media_to_download(self, request, info):self._mockcalled.append('media_to_download')if 'result' in request.meta:return request.meta.get('result')- return super(MockedMediaPipeline, self).media_to_download(request, info)+ return super().media_to_download(request, info)def get_media_requests(self, item, info):self._mockcalled.append('get_media_requests')@@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):def media_downloaded(self, response, request, info):self._mockcalled.append('media_downloaded')- return super(MockedMediaPipeline, self).media_downloaded(response, request, info)+ return super().media_downloaded(response, request, info)def media_failed(self, failure, request, info):self._mockcalled.append('media_failed')- return super(MockedMediaPipeline, self).media_failed(failure, request, info)+ return super().media_failed(failure, request, info)def item_completed(self, results, item, info):self._mockcalled.append('item_completed')- item = super(MockedMediaPipeline, self).item_completed(results, item, info)+ item = super().item_completed(results, item, info)item['results'] = resultsreturn itemdiff --git a/tests/test_request_left.py b/tests/test_request_left.pyindex 5cfef8e7d..373b2e49c 100644--- a/tests/test_request_left.py+++ b/tests/test_request_left.py@@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):name = 'signal_catcher'def __init__(self, crawler, url, *args, **kwargs):- super(SignalCatcherSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)crawler.signals.connect(self.on_request_left,signal=request_left_downloader)self.caught_times = 0diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 9d8c201dd..4b15d0fab 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -117,7 +117,7 @@ class BaseRobotParserTest:class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):def setUp(self):from scrapy.robotstxt import PythonRobotParser- super(PythonRobotParserTest, self)._setUp(PythonRobotParser)+ super()._setUp(PythonRobotParser)def test_length_based_precedence(self):raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")@@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):def setUp(self):from scrapy.robotstxt import ReppyRobotParser- super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)+ super()._setUp(ReppyRobotParser)def test_order_based_precedence(self):raise unittest.SkipTest("Reppy does not support order based directives precedence.")@@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):def setUp(self):from scrapy.robotstxt import RerpRobotParser- super(RerpRobotParserTest, self)._setUp(RerpRobotParser)+ super()._setUp(RerpRobotParser)def test_length_based_precedence(self):raise unittest.SkipTest("Rerp does not support length based directives precedence.")@@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):def setUp(self):from scrapy.robotstxt import ProtegoRobotParser- super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)+ super()._setUp(ProtegoRobotParser)def test_order_based_precedence(self):raise unittest.SkipTest("Protego does not support order based directives precedence.")diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 2b6cb0902..512a7460e 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -53,7 +53,7 @@ class MockCrawler(Crawler):JOBDIR=jobdir,DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',)- super(MockCrawler, self).__init__(Spider, settings)+ super().__init__(Spider, settings)self.engine = MockEngine(downloader=MockDownloader())@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):def __init__(self, start_urls):self.start_urls = start_urls- super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')+ super().__init__(name='StartUrlsSpider')def parse(self, response):passdiff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex e032b247c..e449cd706 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):bypass_status_codes = set()def __init__(self, *args, **kwargs):- super(_HttpErrorSpider, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self.start_urls = [self.mockserver.url("/status?n=200"),self.mockserver.url("/status?n=404"),