Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove six.iter* occurrences
Eugenio Lacuesta1aba5136939f
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex f5f8be6e8..73d84664f 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -4,7 +4,6 @@ from time import timefrom datetime import datetimefrom collections import deque-import sixfrom twisted.internet import reactor, defer, taskfrom scrapy.utils.defer import mustbe_deferred@@ -189,7 +188,7 @@ class Downloader(object):def close(self):self._slot_gc_loop.stop()- for slot in six.itervalues(self.slots):+ for slot in self.slots.values():slot.close()def _slot_gc(self, age=60):diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 0b55d32fa..39a0b1f51 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -1,8 +1,9 @@"""Download handlers for different schemes"""import logging+from twisted.internet import defer-import six+from scrapy.exceptions import NotSupported, NotConfiguredfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_object@@ -22,7 +23,7 @@ class DownloadHandlers(object):self._notconfigured = {} # remembers failed handlershandlers = without_none_values(crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))- for scheme, clspath in six.iteritems(handlers):+ for scheme, clspath in handlers.items():self._schemes[scheme] = clspathself._load_handler(scheme, skip_lazy=True)diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 0d2b9900c..9deba40d6 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -1,5 +1,4 @@import os-import siximport loggingfrom collections import defaultdict@@ -8,6 +7,7 @@ from scrapy.http import Responsefrom scrapy.http.cookies import CookieJarfrom scrapy.utils.python import to_unicode+logger = logging.getLogger(__name__)@@ -82,8 +82,10 @@ class CookiesMiddleware(object):def _get_request_cookies(self, jar, request):if isinstance(request.cookies, dict):- cookie_list = [{'name': k, 'value': v} for k, v in \- six.iteritems(request.cookies)]+ cookie_list = [+ {'name': k, 'value': v}+ for k, v in request.cookies.items()+ ]else:cookie_list = request.cookiesdiff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.pyindex e2d73f347..dfd64c35f 100644--- a/scrapy/downloadermiddlewares/decompression.py+++ b/scrapy/downloadermiddlewares/decompression.py@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.import bz2import gzip-from io import BytesIOimport zipfileimport tarfileimport logging+from io import BytesIOfrom tempfile import mktemp-import six-from scrapy.responsetypes import responsetypes+logger = logging.getLogger(__name__)@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):if not response.body:return response- for fmt, func in six.iteritems(self._formats):+ for fmt, func in self._formats.items():new_response = func(response)if new_response:logger.debug('Decompressed response with format: %(responsefmt)s',diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex d9531e67c..19af2d6e4 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -62,9 +62,9 @@ class BaseItemExporter(object):include_empty = self.export_empty_fieldsif self.fields_to_export is None:if include_empty and not isinstance(item, dict):- field_iter = six.iterkeys(item.fields)+ field_iter = item.fields.keys()else:- field_iter = six.iterkeys(item)+ field_iter = item.keys()else:if include_empty:field_iter = self.fields_to_export@@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter):return valuedef _serialize_dict(self, value):- for key, val in six.iteritems(value):+ for key, val in value.items():key = to_bytes(key) if self.binary else keyyield key, self._serialize_value(val)diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.pyindex 263d8ce4c..892aa8a86 100644--- a/scrapy/extensions/memdebug.py+++ b/scrapy/extensions/memdebug.py@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst"""import gc-import sixfrom scrapy import signalsfrom scrapy.exceptions import NotConfigured@@ -28,7 +27,7 @@ class MemoryDebugger(object):def spider_closed(self, spider, reason):gc.collect()self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)- for cls, wdict in six.iteritems(live_refs):+ for cls, wdict in live_refs.items():if not wdict:continueself.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex d2bcf77b7..af02c8484 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rstfrom urllib.parse import urljoin, urlencodeimport lxml.html-import sixfrom parsel.selector import create_root_nodefrom w3lib.html import strip_html5_whitespace@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):# We didn't find it, so now we build an XPath expression out of the other# arguments, because they can be used as suchxpath = u'.//*' + \- u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))+ u''.join(u'[@%s="%s"]' % c for c in clickdata.items())el = form.xpath(xpath)if len(el) == 1:return (el[0].get('name'), el[0].get('value') or '')diff --git a/scrapy/item.py b/scrapy/item.pyindex 32f9b2ebb..4e0f0ac44 100644--- a/scrapy/item.py+++ b/scrapy/item.py@@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem):def __init__(self, *args, **kwargs):self._values = {}if args or kwargs: # avoid creating dict for most common case- for k, v in six.iteritems(dict(*args, **kwargs)):+ for k, v in dict(*args, **kwargs).items():self[k] = vdef __getitem__(self, key):diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.pyindex 60fd6d222..fe01a856f 100644--- a/scrapy/loader/__init__.py+++ b/scrapy/loader/__init__.py@@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst"""from collections import defaultdict-import six-from scrapy.item import Itemfrom scrapy.loader.common import wrap_loader_contextfrom scrapy.loader.processors import Identity@@ -72,7 +70,7 @@ class ItemLoader(object):if value is None:returnif not field_name:- for k, v in six.iteritems(value):+ for k, v in value.items():self._add_value(k, v)else:self._add_value(field_name, value)@@ -82,7 +80,7 @@ class ItemLoader(object):if value is None:returnif not field_name:- for k, v in six.iteritems(value):+ for k, v in value.items():self._replace_value(k, v)else:self._replace_value(field_name, value)diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 432d4c182..6d55c8980 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -8,14 +8,12 @@ import hashlibimport loggingimport mimetypesimport os-import os.pathimport timefrom collections import defaultdictfrom email.utils import parsedate_tz, mktime_tzfrom io import BytesIOfrom urllib.parse import urlparse-import sixfrom twisted.internet import defer, threadsfrom scrapy.pipelines.media import MediaPipeline@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_strfrom scrapy.utils.boto import is_botocorefrom scrapy.utils.datatypes import CaselessDict+logger = logging.getLogger(__name__)@@ -153,14 +152,14 @@ class S3FilesStore(object):Bucket=self.bucket,Key=key_name,Body=buf,- Metadata={k: str(v) for k, v in six.iteritems(meta or {})},+ Metadata={k: str(v) for k, v in (meta or {}).items()},ACL=self.POLICY,**extra)else:b = self._get_boto_bucket()k = b.new_key(key_name)if meta:- for metakey, metavalue in six.iteritems(meta):+ for metakey, metavalue in meta.items():k.set_metadata(metakey, str(metavalue))h = self.HEADERS.copy()if headers:@@ -201,7 +200,7 @@ class S3FilesStore(object):'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',})extra = {}- for key, value in six.iteritems(headers):+ for key, value in headers.items():try:kwarg = mapping[key]except KeyError:@@ -249,7 +248,7 @@ class GCSFilesStore(object):def persist_file(self, path, buf, info, meta=None, headers=None):blob = self.bucket.blob(self.prefix + path)blob.cache_control = self.CACHE_CONTROL- blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}+ blob.metadata = {k: str(v) for k, v in (meta or {}).items()}return threads.deferToThread(blob.upload_from_string,data=buf.getvalue(),diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex e77cef4ff..e9c6b759c 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rstimport functoolsimport hashlibfrom io import BytesIO-import sixfrom PIL import Image@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):image, buf = self.convert_image(orig_image)yield path, image, buf- for thumb_id, size in six.iteritems(self.thumbs):+ for thumb_id, size in self.thumbs.items():thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)thumb_image, thumb_buf = self.convert_image(image, size)yield thumb_path, thumb_image, thumb_bufdiff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.pyindex b64fbbd42..91d309147 100644--- a/scrapy/responsetypes.py+++ b/scrapy/responsetypes.py@@ -5,7 +5,6 @@ based on different criteria.from mimetypes import MimeTypesfrom pkgutil import get_datafrom io import StringIO-import sixfrom scrapy.http import Responsefrom scrapy.utils.misc import load_object@@ -36,7 +35,7 @@ class ResponseTypes(object):self.mimetypes = MimeTypes()mimedata = get_data('scrapy', 'mime.types').decode('utf8')self.mimetypes.readfp(StringIO(mimedata))- for mimetype, cls in six.iteritems(self.CLASSES):+ for mimetype, cls in self.CLASSES.items():self.classes[mimetype] = load_object(cls)def from_mimetype(self, mimetype):diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex c871e86e0..d53b28895 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -317,10 +317,10 @@ class BaseSettings(MutableMapping):values = json.loads(values)if values is not None:if isinstance(values, BaseSettings):- for name, value in six.iteritems(values):+ for name, value in values.items():self.set(name, value, values.getpriority(name))else:- for name, value in six.iteritems(values):+ for name, value in values.items():self.set(name, value, priority)def delete(self, name, priority='project'):@@ -377,7 +377,7 @@ class BaseSettings(MutableMapping):def _to_dict(self):return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)- for k, v in six.iteritems(self)}+ for k, v in self.items()}def copy_to_dict(self):"""@@ -445,7 +445,7 @@ class Settings(BaseSettings):self.setmodule(default_settings, 'default')# Promote default dictionaries to BaseSettings instances for per-key# priorities- for name, val in six.iteritems(self):+ for name, val in self.items():if isinstance(val, dict):self.set(name, BaseSettings(val, 'default'), 'default')self.update(values, priority)diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 561bb72fc..7a15e77ff 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -1,11 +1,9 @@-from configparser import ConfigParserimport osimport sysimport numbers+from configparser import ConfigParserfrom operator import itemgetter-import six-from scrapy.settings import BaseSettingsfrom scrapy.utils.deprecate import update_classpathfrom scrapy.utils.python import without_none_values@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):def _map_keys(compdict):if isinstance(compdict, BaseSettings):compbs = BaseSettings()- for k, v in six.iteritems(compdict):+ for k, v in compdict.items():prio = compdict.getpriority(k)if compbs.getpriority(convert(k)) == prio:raise ValueError('Some paths in {!r} convert to the same '@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):return compbselse:_check_components(compdict)- return {convert(k): v for k, v in six.iteritems(compdict)}+ return {convert(k): v for k, v in compdict.items()}def _validate_values(compdict):"""Fail if a value in the components dict is not a real number or None."""- for name, value in six.iteritems(compdict):+ for name, value in compdict.items():if value is not None and not isinstance(value, numbers.Real):raise ValueError('Invalid value {} for component {}, please provide ' \'a real number or None instead'.format(value, name))@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):_validate_values(compdict)compdict = without_none_values(_map_keys(compdict))- return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]+ return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]def arglist_to_dict(arglist):diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 6e9de47f3..56d4d1b8e 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library.import copyimport collections-from collections.abc import Mappingimport warnings-import six-from scrapy.exceptions import ScrapyDeprecationWarning@@ -151,7 +148,7 @@ class MultiValueDict(dict):self.setlistdefault(key, []).append(value)except TypeError:raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")- for key, value in six.iteritems(kwargs):+ for key, value in kwargs.items():self.setlistdefault(key, []).append(value)@@ -226,7 +223,7 @@ class CaselessDict(dict):return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))def update(self, seq):- seq = seq.items() if isinstance(seq, Mapping) else seq+ seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seqiseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)super(CaselessDict, self).update(iseq)diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex cb8ac3c8e..845c19fb9 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):dict or a list of tuples, like any dict constructor supports."""d = {}- for k, v in six.iteritems(dict(dct_or_tuples)):+ for k, v in dict(dct_or_tuples).items():k = k.encode(encoding) if isinstance(k, six.text_type) else kif not keys_only:v = v.encode(encoding) if isinstance(v, six.text_type) else v@@ -345,7 +345,7 @@ def without_none_values(iterable):value ``None`` have been removed."""try:- return {k: v for k, v in six.iteritems(iterable) if v is not None}+ return {k: v for k, v in iterable.items() if v is not None}except AttributeError:return type(iterable)((v for v in iterable if v is not None))diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 94b24f67e..48ad5041e 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -1,11 +1,10 @@import loggingimport inspect-import six-from scrapy.spiders import Spiderfrom scrapy.utils.misc import arg_to_iter+logger = logging.getLogger(__name__)@@ -21,7 +20,7 @@ def iter_spider_classes(module):# singleton in scrapy.spider.spidersfrom scrapy.spiders import Spider- for obj in six.itervalues(vars(module)):+ for obj in vars(module).values():if inspect.isclass(obj) and \issubclass(obj, Spider) and \obj.__module__ == module.__name__ and \diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.pyindex 78389e464..4842b95df 100644--- a/scrapy/utils/trackref.py+++ b/scrapy/utils/trackref.py@@ -13,7 +13,6 @@ import weakreffrom time import timefrom operator import itemgetterfrom collections import defaultdict-import sixNoneType = type(None)@@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType):"""Return a tabular representation of tracked objects"""s = "Live References\n\n"now = time()- for cls, wdict in sorted(six.iteritems(live_refs),+ for cls, wdict in sorted(live_refs.items(),key=lambda x: x[0].__name__):if not wdict:continueif issubclass(cls, ignore):continue- oldest = min(six.itervalues(wdict))+ oldest = min(wdict.values())s += "%-30s %6d oldest: %ds ago\n" % (cls.__name__, len(wdict), now - oldest)@@ -56,15 +55,15 @@ def print_live_refs(*a, **kw):def get_oldest(class_name):"""Get the oldest object for a specific class name"""- for cls, wdict in six.iteritems(live_refs):+ for cls, wdict in live_refs.items():if cls.__name__ == class_name:if not wdict:break- return min(six.iteritems(wdict), key=itemgetter(1))[0]+ return min(wdict.items(), key=itemgetter(1))[0]def iter_all(class_name):"""Iterate over all objects of the same class by its class name"""- for cls, wdict in six.iteritems(live_refs):+ for cls, wdict in live_refs.items():if cls.__name__ == class_name:- return six.iterkeys(wdict)+ return wdict.keys()diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 32e65bed5..d5cbef6f5 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -10,7 +10,7 @@ from . import default_settingsclass SettingsGlobalFuncsTest(unittest.TestCase):def test_get_settings_priority(self):- for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):+ for prio_str, prio_num in SETTINGS_PRIORITIES.items():self.assertEqual(get_settings_priority(prio_str), prio_num)self.assertEqual(get_settings_priority(99), 99)@@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase):self.settings.setmodule('tests.test_settings.default_settings', 10)- self.assertCountEqual(six.iterkeys(self.settings.attributes),- six.iterkeys(ctrl_attributes))+ self.assertCountEqual(self.settings.attributes.keys(),+ ctrl_attributes.keys())- for key in six.iterkeys(ctrl_attributes):+ for key in ctrl_attributes.keys():attr = self.settings.attributes[key]ctrl_attr = ctrl_attributes[key]self.assertEqual(attr.value, ctrl_attr.value)@@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase):}settings = self.settingssettings.attributes = {key: SettingsAttribute(value, 0) for key, value- in six.iteritems(test_configuration)}+ in test_configuration.items()}self.assertTrue(settings.getbool('TEST_ENABLED1'))self.assertTrue(settings.getbool('TEST_ENABLED2'))diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex a81946490..0c04e7114 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase):def cleanup(passthrough):# Clean up the server which is hanging around not doing# anything.- connected = list(six.iterkeys(self.wrapper.protocols))+ connected = list(self.wrapper.protocols.keys())# There might be nothing here if the server managed to already see# that the connection was lost.if connected: