Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove six type wrappers

Eugenio Lacuestace8e515fa896
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b998e0d..b88df8f64 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -205,7 +205,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
- if isinstance(spidercls, six.string_types):
+ if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2ff089ce9..e45317491 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
- elif isinstance(serialized_value, six.text_type):
+ elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
else:
self.xg.characters(str(serialized_value))
@@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
- if isinstance(value, (six.text_type, bytes)):
+ if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..5bbe7d72a 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -19,7 +19,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
- elif isinstance(value, (six.text_type, bytes)):
+ elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@@ -29,10 +29,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
- elif isinstance(x, six.text_type):
+ elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
- return six.text_type(x).encode(self.encoding)
+ return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index d09eaf849..ff7a44545 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -60,7 +60,7 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
- if not isinstance(url, six.string_types):
+ if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 69bcba577..65400803f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -31,14 +31,14 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
- if isinstance(url, six.text_type):
+ if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
@@ -158,7 +158,7 @@ class TextResponse(Response):
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
- if isinstance(sel.root, six.string_types):
+ if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 623732cee..2fec35799 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser):
ret = []
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 5b7e709de..496ad3053 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag):
- if isinstance(tag, six.string_types):
+ if isinstance(tag, str):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 98bed15e9..a9dffcad3 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d53b28895..c88c9c0e2 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -23,7 +23,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
- if isinstance(priority, six.string_types):
+ if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@@ -173,7 +173,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = value.split(',')
return list(value)
@@ -194,7 +194,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = json.loads(value)
return dict(value)
@@ -284,7 +284,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(module, six.string_types):
+ if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@@ -313,7 +313,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(values, six.string_types):
+ if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 03000ce54..59e2c5661 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -25,7 +25,7 @@ def _identity(request, response):
def _get_method(method, spider):
if callable(method):
return method
- elif isinstance(method, six.string_types):
+ elif isinstance(method, str):
return getattr(spider, method, None)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 534c45c70..2917daf57 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -22,7 +22,7 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
- if isinstance(c, six.string_types):
+ if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +86,7 @@ class SitemapSpider(Spider):
def regex(x):
- if isinstance(x, six.string_types):
+ if isinstance(x, str):
return re.compile(x)
return x
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9693ba768..10481fe8d 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -60,7 +60,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
- self._is_unicode = isinstance(self._text, six.text_type)
+ self._is_unicode = isinstance(self._text, str)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
- expected_types = (Response, six.text_type, six.binary_type)
+ expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
return obj.text
else:
return obj.body.decode('utf-8')
- elif isinstance(obj, six.text_type):
+ elif isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f638adb25..9a44f3576 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicode
from scrapy.item import BaseItem
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
def arg_to_iter(arg):
@@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'):
* if the regex doesn't contain any group the entire regex matching is returned
"""
- if isinstance(regex, six.string_types):
+ if isinstance(regex, str):
regex = re.compile(regex, re.UNICODE)
try:
@@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'):
strings = regex.findall(text) # full regex or numbered groups
strings = flatten(strings)
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 845c19fb9..18fee1964 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -68,7 +68,7 @@ def is_listlike(x):
>>> is_listlike(range(5))
True
"""
- return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+ return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
def unique(list_, key=lambda x: x):
@@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x):
def to_unicode(text, encoding=None, errors='strict'):
"""Return the unicode representation of a bytes object ``text``. If
``text`` is already an unicode object, return it as-is."""
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return text
- if not isinstance(text, (bytes, six.text_type)):
+ if not isinstance(text, (bytes, str)):
raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'):
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
- if not isinstance(text, six.string_types):
+ if not isinstance(text, str):
raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
yield (text[offset:], offset)
yield (text, 0)
- if isinstance(pattern, six.string_types):
+ if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
@@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
"""
d = {}
for k, v in dict(dct_or_tuples).items():
- k = k.encode(encoding) if isinstance(k, six.text_type) else k
+ k = k.encode(encoding) if isinstance(k, str) else k
if not keys_only:
- v = v.encode(encoding) if isinstance(v, six.text_type) else v
+ v = v.encode(encoding) if isinstance(v, str) else v
d[k] = v
return d