Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove six type wrappers
Eugenio Lacuestace8e515fa896
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 19b998e0d..b88df8f64 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -205,7 +205,7 @@ class CrawlerRunner(object):return self._create_crawler(crawler_or_spidercls)def _create_crawler(self, spidercls):- if isinstance(spidercls, six.string_types):+ if isinstance(spidercls, str):spidercls = self.spider_loader.load(spidercls)return Crawler(spidercls, self.settings)diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 2ff089ce9..e45317491 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter):for value in serialized_value:self._export_xml_field('value', value, depth=depth+1)self._beautify_indent(depth=depth)- elif isinstance(serialized_value, six.text_type):+ elif isinstance(serialized_value, str):self.xg.characters(serialized_value)else:self.xg.characters(str(serialized_value))@@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter):if is_listlike(value):return [self._serialize_value(v) for v in value]encode_func = to_bytes if self.binary else to_unicode- if isinstance(value, (six.text_type, bytes)):+ if isinstance(value, (str, bytes)):return encode_func(value, encoding=self.encoding)return valuediff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 62507eb19..5bbe7d72a 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -19,7 +19,7 @@ class Headers(CaselessDict):"""Normalize values to bytes"""if value is None:value = []- elif isinstance(value, (six.text_type, bytes)):+ elif isinstance(value, (str, bytes)):value = [value]elif not hasattr(value, '__iter__'):value = [value]@@ -29,10 +29,10 @@ class Headers(CaselessDict):def _tobytes(self, x):if isinstance(x, bytes):return x- elif isinstance(x, six.text_type):+ elif isinstance(x, str):return x.encode(self.encoding)elif isinstance(x, int):- return six.text_type(x).encode(self.encoding)+ return str(x).encode(self.encoding)else:raise TypeError('Unsupported value type: {}'.format(type(x)))diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex d09eaf849..ff7a44545 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -60,7 +60,7 @@ class Request(object_ref):return self._urldef _set_url(self, url):- if not isinstance(url, six.string_types):+ if not isinstance(url, str):raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)s = safe_url_string(url, self.encoding)diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 69bcba577..65400803f 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -31,14 +31,14 @@ class TextResponse(Response):super(TextResponse, self).__init__(*args, **kwargs)def _set_url(self, url):- if isinstance(url, six.text_type):+ if isinstance(url, str):self._url = to_unicode(url, self.encoding)else:super(TextResponse, self)._set_url(url)def _set_body(self, body):self._body = b'' # used by encoding detection- if isinstance(body, six.text_type):+ if isinstance(body, str):if self._encoding is None:raise TypeError('Cannot convert unicode body - %s has no encoding' %type(self).__name__)@@ -158,7 +158,7 @@ class TextResponse(Response):def _url_from_selector(sel):# type: (parsel.Selector) -> str- if isinstance(sel.root, six.string_types):+ if isinstance(sel.root, str):# e.g. ::attr(href) resultreturn strip_html5_whitespace(sel.root)if not hasattr(sel.root, 'tag'):diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.pyindex 623732cee..2fec35799 100644--- a/scrapy/linkextractors/htmlparser.py+++ b/scrapy/linkextractors/htmlparser.py@@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser):ret = []base_url = urljoin(response_url, self.base_url) if self.base_url else response_urlfor link in links:- if isinstance(link.url, six.text_type):+ if isinstance(link.url, str):link.url = link.url.encode(response_encoding)try:link.url = urljoin(base_url, link.url)diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 5b7e709de..496ad3053 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")def _nons(tag):- if isinstance(tag, six.string_types):+ if isinstance(tag, str):if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:return tag.split('}')[-1]return tagdiff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.pyindex 98bed15e9..a9dffcad3 100644--- a/scrapy/linkextractors/sgml.py+++ b/scrapy/linkextractors/sgml.py@@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser):if base_url is None:base_url = urljoin(response_url, self.base_url) if self.base_url else response_urlfor link in self.links:- if isinstance(link.url, six.text_type):+ if isinstance(link.url, str):link.url = link.url.encode(response_encoding)try:link.url = urljoin(base_url, link.url)diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex d53b28895..c88c9c0e2 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -23,7 +23,7 @@ def get_settings_priority(priority)::attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns itsnumerical value, or directly returns a given numerical priority."""- if isinstance(priority, six.string_types):+ if isinstance(priority, str):return SETTINGS_PRIORITIES[priority]else:return priority@@ -173,7 +173,7 @@ class BaseSettings(MutableMapping)::type default: any"""value = self.get(name, default or [])- if isinstance(value, six.string_types):+ if isinstance(value, str):value = value.split(',')return list(value)@@ -194,7 +194,7 @@ class BaseSettings(MutableMapping)::type default: any"""value = self.get(name, default or {})- if isinstance(value, six.string_types):+ if isinstance(value, str):value = json.loads(value)return dict(value)@@ -284,7 +284,7 @@ class BaseSettings(MutableMapping)::type priority: string or int"""self._assert_mutability()- if isinstance(module, six.string_types):+ if isinstance(module, str):module = import_module(module)for key in dir(module):if key.isupper():@@ -313,7 +313,7 @@ class BaseSettings(MutableMapping)::type priority: string or int"""self._assert_mutability()- if isinstance(values, six.string_types):+ if isinstance(values, str):values = json.loads(values)if values is not None:if isinstance(values, BaseSettings):diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 03000ce54..59e2c5661 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -25,7 +25,7 @@ def _identity(request, response):def _get_method(method, spider):if callable(method):return method- elif isinstance(method, six.string_types):+ elif isinstance(method, str):return getattr(spider, method, None)diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 534c45c70..2917daf57 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -22,7 +22,7 @@ class SitemapSpider(Spider):super(SitemapSpider, self).__init__(*a, **kw)self._cbs = []for r, c in self.sitemap_rules:- if isinstance(c, six.string_types):+ if isinstance(c, str):c = getattr(self, c)self._cbs.append((regex(r), c))self._follow = [regex(x) for x in self.sitemap_follow]@@ -86,7 +86,7 @@ class SitemapSpider(Spider):def regex(x):- if isinstance(x, six.string_types):+ if isinstance(x, str):return re.compile(x)return xdiff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex 9693ba768..10481fe8d 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -60,7 +60,7 @@ class _StreamReader(object):self._text, self.encoding = obj.body, obj.encodingelse:self._text, self.encoding = obj, 'utf-8'- self._is_unicode = isinstance(self._text, six.text_type)+ self._is_unicode = isinstance(self._text, str)def read(self, n=65535):self.read = self._read_unicode if self._is_unicode else self._read_string@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):def _body_or_str(obj, unicode=True):- expected_types = (Response, six.text_type, six.binary_type)+ expected_types = (Response, str, bytes)assert isinstance(obj, expected_types), \"obj must be %s, not %s" % (" or ".join(t.__name__ for t in expected_types),@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):return obj.textelse:return obj.body.decode('utf-8')- elif isinstance(obj, six.text_type):+ elif isinstance(obj, str):return obj if unicode else obj.encode('utf-8')else:return obj.decode('utf-8') if unicode else objdiff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex f638adb25..9a44f3576 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicodefrom scrapy.item import BaseItem-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytesdef arg_to_iter(arg):@@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'):* if the regex doesn't contain any group the entire regex matching is returned"""- if isinstance(regex, six.string_types):+ if isinstance(regex, str):regex = re.compile(regex, re.UNICODE)try:@@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'):strings = regex.findall(text) # full regex or numbered groupsstrings = flatten(strings)- if isinstance(text, six.text_type):+ if isinstance(text, str):return [replace_entities(s, keep=['lt', 'amp']) for s in strings]else:return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 845c19fb9..18fee1964 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -68,7 +68,7 @@ def is_listlike(x):>>> is_listlike(range(5))True"""- return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))+ return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))def unique(list_, key=lambda x: x):@@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x):def to_unicode(text, encoding=None, errors='strict'):"""Return the unicode representation of a bytes object ``text``. If``text`` is already an unicode object, return it as-is."""- if isinstance(text, six.text_type):+ if isinstance(text, str):return text- if not isinstance(text, (bytes, six.text_type)):+ if not isinstance(text, (bytes, str)):raise TypeError('to_unicode must receive a bytes or str ''object, got %s' % type(text).__name__)if encoding is None:@@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'):is already a bytes object, return it as-is."""if isinstance(text, bytes):return text- if not isinstance(text, six.string_types):+ if not isinstance(text, str):raise TypeError('to_bytes must receive a str or bytes ''object, got %s' % type(text).__name__)if encoding is None:@@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024):yield (text[offset:], offset)yield (text, 0)- if isinstance(pattern, six.string_types):+ if isinstance(pattern, str):pattern = re.compile(pattern)for chunk, offset in _chunk_iter():@@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):"""d = {}for k, v in dict(dct_or_tuples).items():- k = k.encode(encoding) if isinstance(k, six.text_type) else k+ k = k.encode(encoding) if isinstance(k, str) else kif not keys_only:- v = v.encode(encoding) if isinstance(v, six.text_type) else v+ v = v.encode(encoding) if isinstance(v, str) else vd[k] = vreturn d