Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Replace to_native_str calls with to_unicode.

Andrey Rakhmatullin397e88355646
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 91b45a8fc..7d917cb74 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
r"""
Return binary content of a CONNECT request.
- >>> from scrapy.utils.python import to_native_str as s
+ >>> from scrapy.utils.python import to_unicode as s
>>> s(tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..0d2b9900c 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -6,7 +6,7 @@ from collections import defaultdict
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
def _debug_cookie(self, request, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
def _debug_set_cookie(self, response, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a5dfb79c..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
import logging
-import sys
-import re
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 40567f53b..f276c28e8 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from six.moves import cPickle as pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):
def _build_row(self, values):
for s in values:
try:
- yield to_native_str(s, self.encoding)
+ yield to_unicode(s, self.encoding)
except TypeError:
yield s
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4e8056750..4532c3ab7 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
)
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
return name in self.request.headers
def get_header(self, name, default=None):
- return to_native_str(self.request.headers.get(name, default),
+ return to_unicode(self.request.headers.get(name, default),
errors='replace')
def header_items(self):
return [
- (to_native_str(k, errors='replace'),
- [to_native_str(x, errors='replace') for x in v])
+ (to_unicode(k, errors='replace'),
+ [to_unicode(x, errors='replace') for x in v])
for k, v in self.request.headers.items()
]
@@ -189,7 +189,7 @@ class WrappedResponse(object):
# python3 cookiejars calls get_all
def get_all(self, name, default=None):
- return [to_native_str(v, errors='replace')
+ return [to_unicode(v, errors='replace')
for v in self.response.headers.getlist(name)]
# python2 cookiejars calls getheaders
getheaders = get_all
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index a8010877c..37f450e54 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
class TextResponse(Response):
@@ -32,7 +32,7 @@ class TextResponse(Response):
def _set_url(self, url):
if isinstance(url, six.text_type):
- self._url = to_native_str(url, self.encoding)
+ self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
@@ -81,11 +81,11 @@ class TextResponse(Response):
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get(b'Content-Type', b'')
- return http_content_type_encoding(to_native_str(content_type))
+ return http_content_type_encoding(to_unicode(content_type))
def _body_inferred_encoding(self):
if self._cached_benc is None:
- content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+ content_type = to_unicode(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
auto_detect_fun=self._auto_detect_fun,
default_encoding=self._DEFAULT_ENCODING)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8f6f93a44..890c019c8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
from scrapy.utils.response import get_base_url
from scrapy.linkextractors import FilteringLinkExtractor
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
url = self.process_attr(attr_val)
if url is None:
continue
- url = to_native_str(url, encoding=response_encoding)
+ url = to_unicode(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4a2d5bf52..de62276c8 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -10,7 +10,7 @@ import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
header """
if content_encoding:
return Response
- mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+ mimetype = to_unicode(content_type).split(';')[0].strip().lower()
return self.from_mimetype(mimetype)
def from_content_disposition(self, content_disposition):
try:
- filename = to_native_str(content_disposition,
+ filename = to_unicode(content_disposition,
encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
filename = filename.strip('"\'')
return self.from_filename(filename)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 189f165d1..95a8c09b8 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,14 +3,14 @@ import logging
from abc import ABCMeta, abstractmethod
from six import with_metaclass
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
try:
if to_native_str_type:
- robotstxt_body = to_native_str(robotstxt_body)
+ robotstxt_body = to_unicode(robotstxt_body)
else:
robotstxt_body = robotstxt_body.decode('utf-8')
except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
return o
def allowed(self, url, user_agent):
- user_agent = to_native_str(user_agent)
- url = to_native_str(url)
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1ddfb37f4..c76e4d5a2 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
from scrapy.utils.url import strip_url
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
if isinstance(resp_or_url, Response):
policy_header = resp_or_url.headers.get('Referrer-Policy')
if policy_header is not None:
- policy_name = to_native_str(policy_header.decode('latin1'))
+ policy_name = to_unicode(policy_header.decode('latin1'))
if policy_name is None:
return self.default_policy()
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index c7ea7b425..495564ac0 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
import six
from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
eb = _get_method(spider, eb)
request_cls = load_object(d['_class']) if '_class' in d else Request
return request_cls(
- url=to_native_str(d['url']),
+ url=to_unicode(d['url']),
callback=cb,
errback=eb,
method=d['method'],
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index fb5af66a2..63d0ae772 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -9,7 +9,7 @@ import weakref
from six.moves.urllib.parse import urlunparse
from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
from w3lib.url import canonicalize_url
from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
referrer = request.headers.get('Referer')
if referrer is None:
return referrer
- return to_native_str(referrer, errors='replace')
+ return to_unicode(referrer, errors='replace')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c3236afd4..feab07431 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -8,7 +8,7 @@ import webbrowser
import tempfile
from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
from w3lib import html
@@ -36,7 +36,7 @@ def response_status_message(status):
"""Return status code plus status text descriptive message
"""
message = http.RESPONSES.get(int(status), "Unknown Status")
- return '%s %s' % (status, to_native_str(message))
+ return '%s %s' % (status, to_unicode(message))
def response_httprepr(response):
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 02aed60ee..6e81b33ff 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -3,7 +3,7 @@
import OpenSSL
import OpenSSL._util as pyOpenSSLutil
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
def ffi_buf_to_string(buf):
- return to_native_str(pyOpenSSLutil.ffi.string(buf))
+ return to_unicode(pyOpenSSLutil.ffi.string(buf))
def x509name_to_string(x509name):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 62d5d76b4..b134beb88 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,17 +1,16 @@
import os
from os.path import join, abspath
-from twisted.trial import unittest
from twisted.internet import defer
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
from tests.test_commands import CommandTest
def _textmode(bstr):
"""Normalize input the same as writing to a file
and reading from it in text mode"""
- return to_native_str(bstr).replace(os.linesep, '\n')
+ return to_unicode(bstr).replace(os.linesep, '\n')
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b8445ae6c..536379170 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -10,13 +10,10 @@ from contextlib import contextmanager
from threading import Timer
from twisted.trial import unittest
-from twisted.internet import defer
import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
from tests.test_crawler import ExceptionSpider, NoRequestsSpider
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
finally:
timer.cancel()
- return p, to_native_str(stdout), to_native_str(stderr)
+ return p, to_unicode(stdout), to_unicode(stderr)
class StartprojectTest(ProjectTest):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 7431f921f..abe2ab557 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
S3FeedStorage, StdoutFeedStorage,
BlockingFeedStorage)
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
class FileFeedStorageTest(unittest.TestCase):
@@ -456,7 +455,7 @@ class FeedExportTest(unittest.TestCase):
settings.update({'FEED_FORMAT': 'csv'})
data = yield self.exported_data(items, settings)
- reader = csv.DictReader(to_native_str(data).splitlines())
+ reader = csv.DictReader(to_unicode(data).splitlines())
got_rows = list(reader)
if ordered:
self.assertEqual(reader.fieldnames, header)
@@ -470,7 +469,7 @@ class FeedExportTest(unittest.TestCase):
settings = settings or {}
settings.update({'FEED_FORMAT': 'jl'})
data = yield self.exported_data(items, settings)
- parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+ parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
rows = [{k: v for k, v in row.items() if v} for row in rows]
self.assertEqual(rows, parsed)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index effb9e53b..807265981 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -7,12 +7,11 @@ from unittest import mock
from urllib.parse import unquote_to_bytes
import warnings
-import six
from six.moves import xmlrpc_client as xmlrpclib
from six.moves.urllib.parse import urlparse, parse_qs, unquote
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
class RequestTest(unittest.TestCase):
@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):
request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):
- first = to_native_str(first).split("&")
- second = to_native_str(second).split("&")
+ first = to_unicode(first).split("&")
+ second = to_unicode(second).split("&")
return self.assertEqual(sorted(first), sorted(second), msg)
def test_empty_formdata(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index d6e77d6b8..80bf51647 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
XmlResponse, Headers)
from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
from scrapy.exceptions import NotSupported
from scrapy.link import Link
from tests import get_testdata
@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
assert isinstance(resp.url, str)
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
- self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+ self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
- self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+ self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9aaab560a..cd7480e33 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,6 +1,5 @@
# coding=utf-8
from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
def reppy_available():