Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace to_native_str calls with to_unicode.
Andrey Rakhmatullina138fb05d4f0
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 91b45a8fc..7d917cb74 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):r"""Return binary content of a CONNECT request.- >>> from scrapy.utils.python import to_native_str as s+ >>> from scrapy.utils.python import to_unicode as s>>> s(tunnel_request_data("example.com", 8080))'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'>>> s(tunnel_request_data("example.com", 8080, b"123"))diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 321c0171b..0d2b9900c 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -6,7 +6,7 @@ from collections import defaultdictfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Responsefrom scrapy.http.cookies import CookieJar-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicodelogger = logging.getLogger(__name__)@@ -53,7 +53,7 @@ class CookiesMiddleware(object):def _debug_cookie(self, request, spider):if self.debug:- cl = [to_native_str(c, errors='replace')+ cl = [to_unicode(c, errors='replace')for c in request.headers.getlist('Cookie')]if cl:cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)@@ -62,7 +62,7 @@ class CookiesMiddleware(object):def _debug_set_cookie(self, response, spider):if self.debug:- cl = [to_native_str(c, errors='replace')+ cl = [to_unicode(c, errors='replace')for c in response.headers.getlist('Set-Cookie')]if cl:cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 6a5dfb79c..251706c50 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting."""import logging-import sys-import refrom twisted.internet.defer import Deferred, maybeDeferredfrom scrapy.exceptions import NotConfigured, IgnoreRequestfrom scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_info-from scrapy.utils.python import to_native_strfrom scrapy.utils.misc import load_objectlogger = logging.getLogger(__name__)diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 1aa195b0b..8eb52995e 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -12,7 +12,7 @@ from six.moves import cPickle as picklefrom xml.sax.saxutils import XMLGeneratorfrom scrapy.utils.serialize import ScrapyJSONEncoder-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike+from scrapy.utils.python import to_bytes, to_unicode, is_listlikefrom scrapy.item import BaseItemfrom scrapy.exceptions import ScrapyDeprecationWarningimport warnings@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):def _build_row(self, values):for s in values:try:- yield to_native_str(s, self.encoding)+ yield to_unicode(s, self.encoding)except TypeError:yield sdiff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex 4e8056750..4532c3ab7 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE)from scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicodeclass CookieJar(object):@@ -165,13 +165,13 @@ class WrappedRequest(object):return name in self.request.headersdef get_header(self, name, default=None):- return to_native_str(self.request.headers.get(name, default),+ return to_unicode(self.request.headers.get(name, default),errors='replace')def header_items(self):return [- (to_native_str(k, errors='replace'),- [to_native_str(x, errors='replace') for x in v])+ (to_unicode(k, errors='replace'),+ [to_unicode(x, errors='replace') for x in v])for k, v in self.request.headers.items()]@@ -189,7 +189,7 @@ class WrappedResponse(object):# python3 cookiejars calls get_alldef get_all(self, name, default=None):- return [to_native_str(v, errors='replace')+ return [to_unicode(v, errors='replace')for v in self.response.headers.getlist(name)]# python2 cookiejars calls getheadersgetheaders = get_alldiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex a8010877c..37f450e54 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespacefrom scrapy.http.request import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.response import get_base_url-from scrapy.utils.python import memoizemethod_noargs, to_native_str+from scrapy.utils.python import memoizemethod_noargs, to_unicodeclass TextResponse(Response):@@ -32,7 +32,7 @@ class TextResponse(Response):def _set_url(self, url):if isinstance(url, six.text_type):- self._url = to_native_str(url, self.encoding)+ self._url = to_unicode(url, self.encoding)else:super(TextResponse, self)._set_url(url)@@ -81,11 +81,11 @@ class TextResponse(Response):@memoizemethod_noargsdef _headers_encoding(self):content_type = self.headers.get(b'Content-Type', b'')- return http_content_type_encoding(to_native_str(content_type))+ return http_content_type_encoding(to_unicode(content_type))def _body_inferred_encoding(self):if self._cached_benc is None:- content_type = to_native_str(self.headers.get(b'Content-Type', b''))+ content_type = to_unicode(self.headers.get(b'Content-Type', b''))benc, ubody = html_to_unicode(content_type, self.body,auto_detect_fun=self._auto_detect_fun,default_encoding=self._DEFAULT_ENCODING)diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 8f6f93a44..890c019c8 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_urlfrom scrapy.link import Linkfrom scrapy.utils.misc import arg_to_iter, rel_has_nofollow-from scrapy.utils.python import unique as unique_list, to_native_str+from scrapy.utils.python import unique as unique_list, to_unicodefrom scrapy.utils.response import get_base_urlfrom scrapy.linkextractors import FilteringLinkExtractor@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):url = self.process_attr(attr_val)if url is None:continue- url = to_native_str(url, encoding=response_encoding)+ url = to_unicode(url, encoding=response_encoding)# to fix relative links after process_valueurl = urljoin(response_url, url)link = Link(url, _collect_string_content(el) or u'',diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.pyindex 4a2d5bf52..de62276c8 100644--- a/scrapy/responsetypes.py+++ b/scrapy/responsetypes.py@@ -10,7 +10,7 @@ import sixfrom scrapy.http import Responsefrom scrapy.utils.misc import load_object-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str+from scrapy.utils.python import binary_is_text, to_bytes, to_unicodeclass ResponseTypes(object):@@ -55,12 +55,12 @@ class ResponseTypes(object):header """if content_encoding:return Response- mimetype = to_native_str(content_type).split(';')[0].strip().lower()+ mimetype = to_unicode(content_type).split(';')[0].strip().lower()return self.from_mimetype(mimetype)def from_content_disposition(self, content_disposition):try:- filename = to_native_str(content_disposition,+ filename = to_unicode(content_disposition,encoding='latin-1', errors='replace').split(';')[1].split('=')[1]filename = filename.strip('"\'')return self.from_filename(filename)diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 189f165d1..95a8c09b8 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -3,14 +3,14 @@ import loggingfrom abc import ABCMeta, abstractmethodfrom six import with_metaclass-from scrapy.utils.python import to_native_str, to_unicode+from scrapy.utils.python import to_unicodelogger = logging.getLogger(__name__)def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):try:if to_native_str_type:- robotstxt_body = to_native_str(robotstxt_body)+ robotstxt_body = to_unicode(robotstxt_body)else:robotstxt_body = robotstxt_body.decode('utf-8')except UnicodeDecodeError:@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):return odef allowed(self, url, user_agent):- user_agent = to_native_str(user_agent)- url = to_native_str(url)+ user_agent = to_unicode(user_agent)+ url = to_unicode(url)return self.rp.can_fetch(user_agent, url)diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 1ddfb37f4..c76e4d5a2 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -10,8 +10,7 @@ from w3lib.url import safe_url_stringfrom scrapy.http import Request, Responsefrom scrapy.exceptions import NotConfiguredfrom scrapy import signals-from scrapy.utils.python import to_native_str-from scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.python import to_unicodefrom scrapy.utils.misc import load_objectfrom scrapy.utils.url import strip_url@@ -322,7 +321,7 @@ class RefererMiddleware(object):if isinstance(resp_or_url, Response):policy_header = resp_or_url.headers.get('Referrer-Policy')if policy_header is not None:- policy_name = to_native_str(policy_header.decode('latin1'))+ policy_name = to_unicode(policy_header.decode('latin1'))if policy_name is None:return self.default_policy()diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.pyindex c7ea7b425..495564ac0 100644--- a/scrapy/utils/reqser.py+++ b/scrapy/utils/reqser.py@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.import sixfrom scrapy.http import Request-from scrapy.utils.python import to_unicode, to_native_str+from scrapy.utils.python import to_unicodefrom scrapy.utils.misc import load_object@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):eb = _get_method(spider, eb)request_cls = load_object(d['_class']) if '_class' in d else Requestreturn request_cls(- url=to_native_str(d['url']),+ url=to_unicode(d['url']),callback=cb,errback=eb,method=d['method'],diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex fb5af66a2..63d0ae772 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -9,7 +9,7 @@ import weakreffrom six.moves.urllib.parse import urlunparsefrom w3lib.http import basic_auth_header-from scrapy.utils.python import to_bytes, to_native_str+from scrapy.utils.python import to_bytes, to_unicodefrom w3lib.url import canonicalize_urlfrom scrapy.utils.httpobj import urlparse_cached@@ -97,4 +97,4 @@ def referer_str(request):referrer = request.headers.get('Referer')if referrer is None:return referrer- return to_native_str(referrer, errors='replace')+ return to_unicode(referrer, errors='replace')diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex c3236afd4..feab07431 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -8,7 +8,7 @@ import webbrowserimport tempfilefrom twisted.web import http-from scrapy.utils.python import to_bytes, to_native_str+from scrapy.utils.python import to_bytes, to_unicodefrom w3lib import html@@ -36,7 +36,7 @@ def response_status_message(status):"""Return status code plus status text descriptive message"""message = http.RESPONSES.get(int(status), "Unknown Status")- return '%s %s' % (status, to_native_str(message))+ return '%s %s' % (status, to_unicode(message))def response_httprepr(response):diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.pyindex 02aed60ee..6e81b33ff 100644--- a/scrapy/utils/ssl.py+++ b/scrapy/utils/ssl.py@@ -3,7 +3,7 @@import OpenSSLimport OpenSSL._util as pyOpenSSLutil-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicode# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)def ffi_buf_to_string(buf):- return to_native_str(pyOpenSSLutil.ffi.string(buf))+ return to_unicode(pyOpenSSLutil.ffi.string(buf))def x509name_to_string(x509name):diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 62d5d76b4..b134beb88 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -1,17 +1,16 @@import osfrom os.path import join, abspath-from twisted.trial import unittestfrom twisted.internet import deferfrom scrapy.utils.testsite import SiteTestfrom scrapy.utils.testproc import ProcessTest-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicodefrom tests.test_commands import CommandTestdef _textmode(bstr):"""Normalize input the same as writing to a fileand reading from it in text mode"""- return to_native_str(bstr).replace(os.linesep, '\n')+ return to_unicode(bstr).replace(os.linesep, '\n')class ParseCommandTest(ProcessTest, SiteTest, CommandTest):command = 'parse'diff --git a/tests/test_commands.py b/tests/test_commands.pyindex b8445ae6c..536379170 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -10,13 +10,10 @@ from contextlib import contextmanagerfrom threading import Timerfrom twisted.trial import unittest-from twisted.internet import deferimport scrapy-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_testenv-from scrapy.utils.testsite import SiteTest-from scrapy.utils.testproc import ProcessTestfrom tests.test_crawler import ExceptionSpider, NoRequestsSpider@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):finally:timer.cancel()- return p, to_native_str(stdout), to_native_str(stderr)+ return p, to_unicode(stdout), to_unicode(stderr)class StartprojectTest(ProjectTest):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 0c70bf80e..87139e81f 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (S3FeedStorage, StdoutFeedStorage,BlockingFeedStorage)from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler-from scrapy.utils.python import to_native_str-from scrapy.utils.project import get_project_settings+from scrapy.utils.python import to_unicodefrom pathlib import Path@@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase):settings.update({'FEED_FORMAT': 'csv'})data = yield self.exported_data(items, settings)- reader = csv.DictReader(to_native_str(data).splitlines())+ reader = csv.DictReader(to_unicode(data).splitlines())got_rows = list(reader)if ordered:self.assertEqual(reader.fieldnames, header)@@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase):settings = settings or {}settings.update({'FEED_FORMAT': 'jl'})data = yield self.exported_data(items, settings)- parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]+ parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]rows = [{k: v for k, v in row.items() if v} for row in rows]self.assertEqual(rows, parsed)diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 9fe201579..3518da21c 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -7,12 +7,11 @@ from unittest import mockfrom urllib.parse import unquote_to_bytesimport warnings-import sixfrom six.moves import xmlrpc_client as xmlrpclibfrom six.moves.urllib.parse import urlparse, parse_qs, unquotefrom scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse-from scrapy.utils.python import to_bytes, to_native_str+from scrapy.utils.python import to_bytes, to_unicodeclass RequestTest(unittest.TestCase):@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):request_class = FormRequestdef assertQueryEqual(self, first, second, msg=None):- first = to_native_str(first).split("&")- second = to_native_str(second).split("&")+ first = to_unicode(first).split("&")+ second = to_unicode(second).split("&")return self.assertEqual(sorted(first), sorted(second), msg)def test_empty_formdata(self):diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex ec3b51086..883c943da 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encodingfrom scrapy.http import (Request, Response, TextResponse, HtmlResponse,XmlResponse, Headers)from scrapy.selector import Selector-from scrapy.utils.python import to_native_str+from scrapy.utils.python import to_unicodefrom scrapy.exceptions import NotSupportedfrom scrapy.link import Linkfrom tests import get_testdata@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):assert isinstance(resp.url, str)resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')- self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))+ self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})- self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))+ self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 9aaab560a..cd7480e33 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -1,6 +1,5 @@# coding=utf-8from twisted.trial import unittest-from scrapy.utils.python import to_native_strdef reppy_available():