Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove six.iter* occurrences

Eugenio Lacuesta1aba5136939f
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index f5f8be6e8..73d84664f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,7 +4,6 @@ from time import time
from datetime import datetime
from collections import deque
-import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
@@ -189,7 +188,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
- for slot in six.itervalues(self.slots):
+ for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
+
from twisted.internet import defer
-import six
+
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
- for scheme, clspath in six.iteritems(handlers):
+ for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 0d2b9900c..9deba40d6 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,5 +1,4 @@
import os
-import six
import logging
from collections import defaultdict
@@ -8,6 +7,7 @@ from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.python import to_unicode
+
logger = logging.getLogger(__name__)
@@ -82,8 +82,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
- cookie_list = [{'name': k, 'value': v} for k, v in \
- six.iteritems(request.cookies)]
+ cookie_list = [
+ {'name': k, 'value': v}
+ for k, v in request.cookies.items()
+ ]
else:
cookie_list = request.cookies
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e2d73f347..dfd64c35f 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
-from io import BytesIO
import zipfile
import tarfile
import logging
+from io import BytesIO
from tempfile import mktemp
-import six
-
from scrapy.responsetypes import responsetypes
+
logger = logging.getLogger(__name__)
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
- for fmt, func in six.iteritems(self._formats):
+ for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index d9531e67c..19af2d6e4 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -62,9 +62,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
- field_iter = six.iterkeys(item.fields)
+ field_iter = item.fields.keys()
else:
- field_iter = six.iterkeys(item)
+ field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter):
return value
def _serialize_dict(self, value):
- for key, val in six.iteritems(value):
+ for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
-import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d2bcf77b7..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
from urllib.parse import urljoin, urlencode
import lxml.html
-import six
from parsel.selector import create_root_node
from w3lib.html import strip_html5_whitespace
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
- u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+ u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/item.py b/scrapy/item.py
index 32f9b2ebb..4e0f0ac44 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem):
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
- for k, v in six.iteritems(dict(*args, **kwargs)):
+ for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 60fd6d222..fe01a856f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst
"""
from collections import defaultdict
-import six
-
from scrapy.item import Item
from scrapy.loader.common import wrap_loader_context
from scrapy.loader.processors import Identity
@@ -72,7 +70,7 @@ class ItemLoader(object):
if value is None:
return
if not field_name:
- for k, v in six.iteritems(value):
+ for k, v in value.items():
self._add_value(k, v)
else:
self._add_value(field_name, value)
@@ -82,7 +80,7 @@ class ItemLoader(object):
if value is None:
return
if not field_name:
- for k, v in six.iteritems(value):
+ for k, v in value.items():
self._replace_value(k, v)
else:
self._replace_value(field_name, value)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 432d4c182..6d55c8980 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -8,14 +8,12 @@ import hashlib
import logging
import mimetypes
import os
-import os.path
import time
from collections import defaultdict
from email.utils import parsedate_tz, mktime_tz
from io import BytesIO
from urllib.parse import urlparse
-import six
from twisted.internet import defer, threads
from scrapy.pipelines.media import MediaPipeline
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
+
logger = logging.getLogger(__name__)
@@ -153,14 +152,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
- Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+ Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
- for metakey, metavalue in six.iteritems(meta):
+ for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
- for key, value in six.iteritems(headers):
+ for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
- blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+ blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e77cef4ff..e9c6b759c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
import functools
import hashlib
from io import BytesIO
-import six
from PIL import Image
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
- for thumb_id, size in six.iteritems(self.thumbs):
+ for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index b64fbbd42..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -5,7 +5,6 @@ based on different criteria.
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
-import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
@@ -36,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
- for mimetype, cls in six.iteritems(self.CLASSES):
+ for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c871e86e0..d53b28895 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -317,10 +317,10 @@ class BaseSettings(MutableMapping):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@@ -377,7 +377,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
- for k, v in six.iteritems(self)}
+ for k, v in self.items()}
def copy_to_dict(self):
"""
@@ -445,7 +445,7 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
- for name, val in six.iteritems(self):
+ for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 561bb72fc..7a15e77ff 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
-from configparser import ConfigParser
import os
import sys
import numbers
+from configparser import ConfigParser
from operator import itemgetter
-import six
-
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
- for k, v in six.iteritems(compdict):
+ for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
return compbs
else:
_check_components(compdict)
- return {convert(k): v for k, v in six.iteritems(compdict)}
+ return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
- for name, value in six.iteritems(compdict):
+ for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real):
raise ValueError('Invalid value {} for component {}, please provide ' \
'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
- return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+ return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
def arglist_to_dict(arglist):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 6e9de47f3..56d4d1b8e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library.
import copy
import collections
-from collections.abc import Mapping
import warnings
-import six
-
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -151,7 +148,7 @@ class MultiValueDict(dict):
self.setlistdefault(key, []).append(value)
except TypeError:
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
- for key, value in six.iteritems(kwargs):
+ for key, value in kwargs.items():
self.setlistdefault(key, []).append(value)
@@ -226,7 +223,7 @@ class CaselessDict(dict):
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
def update(self, seq):
- seq = seq.items() if isinstance(seq, Mapping) else seq
+ seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super(CaselessDict, self).update(iseq)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index cb8ac3c8e..845c19fb9 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
dict or a list of tuples, like any dict constructor supports.
"""
d = {}
- for k, v in six.iteritems(dict(dct_or_tuples)):
+ for k, v in dict(dct_or_tuples).items():
k = k.encode(encoding) if isinstance(k, six.text_type) else k
if not keys_only:
v = v.encode(encoding) if isinstance(v, six.text_type) else v
@@ -345,7 +345,7 @@ def without_none_values(iterable):
value ``None`` have been removed.
"""
try:
- return {k: v for k, v in six.iteritems(iterable) if v is not None}
+ return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 94b24f67e..48ad5041e 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,10 @@
import logging
import inspect
-import six
-
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
+
logger = logging.getLogger(__name__)
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
# singleton in scrapy.spider.spiders
from scrapy.spiders import Spider
- for obj in six.itervalues(vars(module)):
+ for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, Spider) and \
obj.__module__ == module.__name__ and \
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 78389e464..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -13,7 +13,6 @@ import weakref
from time import time
from operator import itemgetter
from collections import defaultdict
-import six
NoneType = type(None)
@@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType):
"""Return a tabular representation of tracked objects"""
s = "Live References\n\n"
now = time()
- for cls, wdict in sorted(six.iteritems(live_refs),
+ for cls, wdict in sorted(live_refs.items(),
key=lambda x: x[0].__name__):
if not wdict:
continue
if issubclass(cls, ignore):
continue
- oldest = min(six.itervalues(wdict))
+ oldest = min(wdict.values())
s += "%-30s %6d oldest: %ds ago\n" % (
cls.__name__, len(wdict), now - oldest
)
@@ -56,15 +55,15 @@ def print_live_refs(*a, **kw):
def get_oldest(class_name):
"""Get the oldest object for a specific class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
if not wdict:
break
- return min(six.iteritems(wdict), key=itemgetter(1))[0]
+ return min(wdict.items(), key=itemgetter(1))[0]
def iter_all(class_name):
"""Iterate over all objects of the same class by its class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
- return six.iterkeys(wdict)
+ return wdict.keys()
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 32e65bed5..d5cbef6f5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -10,7 +10,7 @@ from . import default_settings
class SettingsGlobalFuncsTest(unittest.TestCase):
def test_get_settings_priority(self):
- for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+ for prio_str, prio_num in SETTINGS_PRIORITIES.items():
self.assertEqual(get_settings_priority(prio_str), prio_num)
self.assertEqual(get_settings_priority(99), 99)
@@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase):
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
- self.assertCountEqual(six.iterkeys(self.settings.attributes),
- six.iterkeys(ctrl_attributes))
+ self.assertCountEqual(self.settings.attributes.keys(),
+ ctrl_attributes.keys())
- for key in six.iterkeys(ctrl_attributes):
+ for key in ctrl_attributes.keys():
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase):
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
- in six.iteritems(test_configuration)}
+ in test_configuration.items()}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..0c04e7114 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase):
def cleanup(passthrough):
# Clean up the server which is hanging around not doing
# anything.
- connected = list(six.iterkeys(self.wrapper.protocols))
+ connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected: