Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove the u prefix from strings
Adrián Chaves890b2138a605
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.pyindex 192123473..640660943 100644--- a/docs/_ext/scrapydocs.py+++ b/docs/_ext/scrapydocs.py@@ -17,7 +17,7 @@ class SettingsListDirective(Directive):def is_setting_index(node):if node.tagname == 'index':# index entries for setting directives look like:- # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]+ # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]entry_type, info, refid = node['entries'][0][:3]return entry_type == 'pair' and info.endswith('; setting')return Falsediff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rstindex d0eeb4097..29d9c5805 100644--- a/docs/topics/loaders.rst+++ b/docs/topics/loaders.rst@@ -237,10 +237,10 @@ metadata. Here is an example::>>> from scrapy.loader import ItemLoader>>> il = ItemLoader(item=Product())->>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])->>> il.add_value('price', [u'€', u'<span>1000</span>'])+>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])+>>> il.add_value('price', ['€', '<span>1000</span>'])>>> il.load_item()-{'name': u'Welcome to my website', 'price': u'1000'}+{'name': 'Welcome to my website', 'price': '1000'}The precedence order, for both input and output processors, is as follows:diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rstindex bb46ea80f..5014df6ac 100644--- a/docs/topics/selectors.rst+++ b/docs/topics/selectors.rst@@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath'sExample selecting links in list item with a "class" attribute ending with a digit:>>> from scrapy import Selector->>> doc = u"""+>>> doc = """... <div>... <ul>... <li class="item-0"><a href="link1.html">first item</a></li>@@ -765,7 +765,7 @@ extracting text elements for example.Example extracting microdata (sample content taken from https://schema.org/Product)with groups of itemscopes and corresponding itemprops::- >>> doc = u"""+ >>> doc = """... <div itemscope itemtype="http://schema.org/Product">... <span itemprop="name">Kenmore White 17" Microwave</span>... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.pyindex 9acfc3b23..95a3f17d5 100755--- a/docs/utils/linkfix.py+++ b/docs/utils/linkfix.py@@ -23,7 +23,7 @@ def main():_contents = None# A regex that matches standard linkcheck output lines- line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')+ line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')# Read lines from the linkcheck output filetry:diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 0e6ceef0b..a260798ac 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',namespaces={"re": "http://exslt.org/regular-expressions"})- values = [(k, u'' if v is None else v)+ values = [(k, '' if v is None else v)for k, v in (_value(e) for e in inputs)if k and k not in formdata_keys]@@ -168,7 +168,7 @@ def _select_value(ele, n, v):# This is a workround to bug in lxml fixed 2.3.1# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139selected_options = ele.xpath('.//option[@selected]')- v = [(o.get('value') or o.text or u'').strip() for o in selected_options]+ v = [(o.get('value') or o.text or '').strip() for o in selected_options]return n, v@@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):# We didn't find it, so now we build an XPath expression out of the other# arguments, because they can be used as such- xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())+ xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())el = form.xpath(xpath)if len(el) == 1:return (el[0].get('name'), el[0].get('value') or '')diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 1615d44d7..8b9f961ee 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:url = safe_url_string(url, encoding=response_encoding)# to fix relative links after process_valueurl = urljoin(response_url, url)- link = Link(url, _collect_string_content(el) or u'',+ link = Link(url, _collect_string_content(el) or '',nofollow=rel_has_nofollow(el.get('rel')))links.append(link)return self._deduplicate_if_needed(links)diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 219145f13..0f9e6f1cb 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -44,7 +44,7 @@ class LogFormatter:def dropped(self, item, exception, response, spider):return {'level': logging.INFO, # lowering the level from logging.WARNING- 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",+ 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",'args': {'exception': exception,'item': item,diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.pyindex da99a6be8..591075a98 100644--- a/tests/test_cmdline/__init__.py+++ b/tests/test_cmdline/__init__.py@@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):'EXTENSIONS=' + json.dumps(EXTENSIONS))# XXX: There's gotta be a smarter way to do this...self.assertNotIn("...", settingsstr)- for char in ("'", "<", ">", 'u"'):+ for char in ("'", "<", ">"):settingsstr = settingsstr.replace(char, '"')settingsdict = json.loads(settingsstr)self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 51deb20f4..57d4cdd6b 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):def test_default_mediatype(self):def _test(response):- self.assertEqual(response.text, u'\u038e\u03a3\u038e')+ self.assertEqual(response.text, '\u038e\u03a3\u038e')self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))self.assertEqual(response.encoding, "iso-8859-7")@@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):def test_text_charset(self):def _test(response):- self.assertEqual(response.text, u'\u038e\u03a3\u038e')+ self.assertEqual(response.text, '\u038e\u03a3\u038e')self.assertEqual(response.body, b'\xbe\xd3\xbe')self.assertEqual(response.encoding, "iso-8859-7")@@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):def test_mediatype_parameters(self):def _test(response):- self.assertEqual(response.text, u'\u038e\u03a3\u038e')+ self.assertEqual(response.text, '\u038e\u03a3\u038e')self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))self.assertEqual(response.encoding, "utf-8")diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 9ccc2110b..010577415 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):def test_request_cookies_encoding(self):# 1) UTF8-encoded bytes- req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})+ req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})assert self.mw.process_request(req1, self.spider) is Noneself.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')# 2) Non UTF8-encoded bytes- req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})+ req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})assert self.mw.process_request(req2, self.spider) is Noneself.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')- # 3) Unicode string- req3 = Request('http://example.org', cookies={'a': u'á'})+ # 3) String+ req3 = Request('http://example.org', cookies={'a': 'á'})assert self.mw.process_request(req3, self.spider) is Noneself.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')def test_request_headers_cookie_encoding(self):# 1) UTF8-encoded bytes- req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})+ req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})assert self.mw.process_request(req1, self.spider) is Noneself.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')# 2) Non UTF8-encoded bytes- req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})+ req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})assert self.mw.process_request(req2, self.spider) is Noneself.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')- # 3) Unicode string- req3 = Request('http://example.org', headers={'Cookie': u'a=á'})+ # 3) String+ req3 = Request('http://example.org', headers={'Cookie': 'a=á'})assert self.mw.process_request(req3, self.spider) is Noneself.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 9841d7a76..351631eb8 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):def test_proxy_auth_encoding(self):# utf-8 encoding- os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'+ os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'mw = HttpProxyMiddleware(auth_encoding='utf-8')req = Request('http://scrapytest.org')assert mw.process_request(req, spider) is None@@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')# proxy from request.meta- req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})+ req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})assert mw.process_request(req, spider) is Noneself.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')@@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')# proxy from request.meta, latin-1 encoding- req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})+ req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})assert mw.process_request(req, spider) is Noneself.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 919dbed23..131332131 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):def test_latin1_location(self):req = Request('http://scrapytest.org/first')- latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1+ latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)req_result = self.mw.process_response(req, resp, self.spider)perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'@@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):def test_utf8_location(self):req = Request('http://scrapytest.org/first')- utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding+ utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encodingresp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)req_result = self.mw.process_response(req, resp, self.spider)perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'@@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):self.mw = MetaRefreshMiddleware.from_crawler(crawler)def _body(self, interval=5, url='http://example.org/newpage'):- html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""+ html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""return html.format(interval, url).encode('utf-8')def test_priority_adjust(self):diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex b9452a0e7..f9936baba 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):def _get_successful_crawler(self):crawler = self.crawlercrawler.settings.set('ROBOTSTXT_OBEY', True)- ROBOTS = u"""+ ROBOTS = """User-Agent: *Disallow: /admin/Disallow: /static/@@ -56,7 +56,7 @@ Disallow: /some/randome/page.htmlself.assertIgnored(Request('http://site.local/admin/main'), middleware),self.assertIgnored(Request('http://site.local/static/'), middleware),self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),- self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)+ self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)], fireOnOneErrback=True)def test_robotstxt_ready_parser(self):diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 25da54a65..660c99ce1 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):def setUp(self):if self.item_class is None:raise unittest.SkipTest("item class is None")- self.i = self.item_class(name=u'John\xa3', age=u'22')+ self.i = self.item_class(name='John\xa3', age='22')self.output = BytesIO()self.ie = self._get_exporter()@@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):def test_serialize_field(self):a = ItemAdapter(self.i)res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])- self.assertEqual(res, u'John\xa3')+ self.assertEqual(res, 'John\xa3')res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])- self.assertEqual(res, u'22')+ self.assertEqual(res, '22')def test_fields_to_export(self):ie = self._get_exporter(fields_to_export=['name'])- self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])+ self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')_, name = list(ie._get_serialized_fields(self.i))[0]assert isinstance(name, str)- self.assertEqual(name, u'John\xa3')+ self.assertEqual(name, 'John\xa3')def test_field_custom_serializer(self):- i = self.custom_field_item_class(name=u'John\xa3', age=u'22')+ i = self.custom_field_item_class(name='John\xa3', age='22')a = ItemAdapter(i)ie = self._get_exporter()- self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')@@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):PythonItemExporter(invalid_option='something')def test_nested_item(self):- i1 = self.item_class(name=u'Joseph', age='22')- i2 = dict(name=u'Maria', age=i1)- i3 = self.item_class(name=u'Jesus', age=i2)+ i1 = self.item_class(name='Joseph', age='22')+ i2 = dict(name='Maria', age=i1)+ i3 = self.item_class(name='Jesus', age=i2)ie = self._get_exporter()exported = ie.export_item(i3)self.assertEqual(type(exported), dict)self.assertEqual(exported,- {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}+ {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'})self.assertEqual(type(exported['age']), dict)self.assertEqual(type(exported['age']['age']), dict)def test_export_list(self):- i1 = self.item_class(name=u'Joseph', age='22')- i2 = self.item_class(name=u'Maria', age=[i1])- i3 = self.item_class(name=u'Jesus', age=[i2])+ i1 = self.item_class(name='Joseph', age='22')+ i2 = self.item_class(name='Maria', age=[i1])+ i3 = self.item_class(name='Jesus', age=[i2])ie = self._get_exporter()exported = ie.export_item(i3)self.assertEqual(exported,- {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}+ {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'})self.assertEqual(type(exported['age'][0]), dict)self.assertEqual(type(exported['age'][0]['age'][0]), dict)def test_export_item_dict_list(self):- i1 = self.item_class(name=u'Joseph', age='22')- i2 = dict(name=u'Maria', age=[i1])- i3 = self.item_class(name=u'Jesus', age=[i2])+ i1 = self.item_class(name='Joseph', age='22')+ i2 = dict(name='Maria', age=[i1])+ i3 = self.item_class(name='Jesus', age=[i2])ie = self._get_exporter()exported = ie.export_item(i3)self.assertEqual(exported,- {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}+ {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'})self.assertEqual(type(exported['age'][0]), dict)self.assertEqual(type(exported['age'][0]['age'][0]), dict)def test_export_binary(self):exporter = PythonItemExporter(binary=True)- value = self.item_class(name=u'John\xa3', age=u'22')+ value = self.item_class(name='John\xa3', age='22')expected = {b'name': b'John\xc2\xa3', b'age': b'22'}self.assertEqual(expected, exporter.export_item(value))@@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):return self.assertEqual(split_csv(first), split_csv(second), msg=msg)def _check_output(self):- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')+ self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')def assertExportResult(self, item, expected, **kwargs):fp = BytesIO()@@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):def test_multivalued_fields(self):self.assertExportResult(- self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),+ self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),b"""<?xml version="1.0" encoding="utf-8"?>\n<items><item>@@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):)def test_nested_item(self):- i1 = dict(name=u'foo\xa3hoo', age='22')- i2 = dict(name=u'bar', age=i1)- i3 = self.item_class(name=u'buz', age=i2)+ i1 = dict(name='foo\xa3hoo', age='22')+ i2 = dict(name='bar', age=i1)+ i3 = self.item_class(name='buz', age=i2)self.assertExportResult(i3,@@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):)def test_nested_list_item(self):- i1 = dict(name=u'foo')- i2 = dict(name=u'bar', v2={"egg": ["spam"]})- i3 = self.item_class(name=u'buz', age=[i1, i2])+ i1 = dict(name='foo')+ i2 = dict(name='bar', v2={"egg": ["spam"]})+ i3 = self.item_class(name='buz', age=[i1, i2])self.assertExportResult(i3,@@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):class JsonLinesItemExporterTest(BaseItemExporterTest):- _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}+ _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}def _get_exporter(self, **kwargs):return JsonLinesItemExporter(self.output, **kwargs)@@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):self.assertEqual(exported, ItemAdapter(self.i).asdict())def test_nested_item(self):- i1 = self.item_class(name=u'Joseph', age='22')- i2 = dict(name=u'Maria', age=i1)- i3 = self.item_class(name=u'Jesus', age=i2)+ i1 = self.item_class(name='Joseph', age='22')+ i2 = dict(name='Maria', age=i1)+ i3 = self.item_class(name='Jesus', age=i2)self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()@@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):self.assertTwoItemsExported(ItemAdapter(self.i).asdict())def test_nested_item(self):- i1 = self.item_class(name=u'Joseph\xa3', age='22')- i2 = self.item_class(name=u'Maria', age=i1)- i3 = self.item_class(name=u'Jesus', age=i2)+ i1 = self.item_class(name='Joseph\xa3', age='22')+ i2 = self.item_class(name='Maria', age=i1)+ i3 = self.item_class(name='Jesus', age=i2)self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()exported = json.loads(to_unicode(self.output.getvalue()))- expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}+ expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}self.assertEqual(exported, [expected])def test_nested_dict_item(self):- i1 = dict(name=u'Joseph\xa3', age='22')- i2 = self.item_class(name=u'Maria', age=i1)- i3 = dict(name=u'Jesus', age=i2)+ i1 = dict(name='Joseph\xa3', age='22')+ i2 = self.item_class(name='Maria', age=i1)+ i3 = dict(name='Jesus', age=i2)self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()exported = json.loads(to_unicode(self.output.getvalue()))- expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}+ expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}self.assertEqual(exported, [expected])def test_nonstring_types_item(self):@@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):else:return super(CustomItemExporter, self).serialize_field(field, name, value)- i = self.item_class(name=u'John', age='22')+ i = self.item_class(name='John', age='22')a = ItemAdapter(i)ie = CustomItemExporter()self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')- i2 = {'name': u'John', 'age': '22'}+ i2 = {'name': 'John', 'age': '22'}self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex c49b2e92f..b57349848 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_encoding(self):- items = [dict({'foo': u'Test\xd6'})]+ items = [dict({'foo': 'Test\xd6'})]formats = {'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),@@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_multiple_configs(self):- items = [dict({'foo': u'FOO', 'bar': u'BAR'})]+ items = [dict({'foo': 'FOO', 'bar': 'BAR'})]formats = {'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),@@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_multiple_configs(self):- items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]+ items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]formats = {'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),@@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):@defer.inlineCallbacksdef test_batch_item_count_feeds_setting(self):- items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]+ items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]formats = {'json': ['[{"foo": "FOO"}]'.encode('utf-8'),'[{"foo": "FOO1"}]'.encode('utf-8')],diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex cf3fc8496..64ff7a73d 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):assert h.getlist('X-Forwarded-For') is not hlistdef test_encode_utf8(self):- h = Headers({u'key': u'\xa3'}, encoding='utf-8')+ h = Headers({'key': '\xa3'}, encoding='utf-8')key, val = dict(h).popitem()assert isinstance(key, bytes), keyassert isinstance(val[0], bytes), val[0]self.assertEqual(val[0], b'\xc2\xa3')def test_encode_latin1(self):- h = Headers({u'key': u'\xa3'}, encoding='latin1')+ h = Headers({'key': '\xa3'}, encoding='latin1')key, val = dict(h).popitem()self.assertEqual(val[0], b'\xa3')def test_encode_multiple(self):- h = Headers({u'key': [u'\xa3']}, encoding='utf-8')+ h = Headers({'key': ['\xa3']}, encoding='utf-8')key, val = dict(h).popitem()self.assertEqual(val[0], b'\xc2\xa3')diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 63014b22d..f5cf4e798 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):self.assertFalse(p.headers is r.headers)# headers must not be unicode- h = Headers({'key1': u'val1', u'key2': 'val2'})- h[u'newkey'] = u'newval'+ h = Headers({'key1': 'val1', 'key2': 'val2'})+ h['newkey'] = 'newval'for k, v in h.items():self.assertIsInstance(k, bytes)for s in v:@@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")def test_url_encoding(self):- r = self.request_class(url=u"http://www.scrapy.org/price/£")+ r = self.request_class(url="http://www.scrapy.org/price/£")self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")def test_url_encoding_other(self):# encoding affects only query part of URI, not path# path part should always be UTF-8 encoded before percent-escaping- r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")+ r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")- r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")+ r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")def test_url_encoding_query(self):- r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")+ r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")# should be same as above- r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")+ r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")def test_url_encoding_query_latin1(self):# encoding is used for encoding query-string before percent-escaping;# path is still UTF-8 encoded before percent-escaping- r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")+ r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")def test_url_encoding_nonutf8_untouched(self):@@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):# characters. Otherwise, in the future the IRI will be mapped to# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different# URI from "http://www.example.org/r%E9sum%E9.html".- r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")+ r1 = self.request_class(url="http://www.scrapy.org/price/%a3")self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")- r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")+ r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")- r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")+ r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")- r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")+ r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")def test_body(self):@@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):assert isinstance(r2.body, bytes)self.assertEqual(r2.encoding, 'utf-8') # default encoding- r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')+ r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')assert isinstance(r3.body, bytes)self.assertEqual(r3.body, b"Price: \xc2\xa3100")- r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')+ r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')assert isinstance(r4.body, bytes)self.assertEqual(r4.body, b"Price: \xa3100")@@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):r = self.request_class(url="http://www.example.com/ajax.html#!key=value")self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")# unicode url- r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")+ r = self.request_class(url="http://www.example.com/ajax.html#!key=value")self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")def test_copy(self):@@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):assert r4.dont_filter is Falsedef test_method_always_str(self):- r = self.request_class("http://www.example.com", method=u"POST")+ r = self.request_class("http://www.example.com", method="POST")assert isinstance(r.method, str)def test_immutable_attributes(self):@@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):def test_default_encoding_textual_data(self):# using default encoding (utf-8)- data = {u'µ one': u'two', u'price': u'£ 100'}+ data = {'µ one': 'two', 'price': '£ 100'}r2 = self.request_class("http://www.example.com", formdata=data)self.assertEqual(r2.method, 'POST')self.assertEqual(r2.encoding, 'utf-8')@@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):def test_default_encoding_mixed_data(self):# using default encoding (utf-8)- data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}+ data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}r2 = self.request_class("http://www.example.com", formdata=data)self.assertEqual(r2.method, 'POST')self.assertEqual(r2.encoding, 'utf-8')@@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')def test_custom_encoding_textual_data(self):- data = {'price': u'£ 100'}+ data = {'price': '£ 100'}r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')self.assertEqual(r3.encoding, 'latin1')self.assertEqual(r3.body, b'price=%A3+100')def test_multi_key_values(self):# using multiples values for a single key- data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}+ data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}r3 = self.request_class("http://www.example.com", formdata=data)self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')@@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')self.assertEqual(req.url, "http://www.example.com/this/post.php")fs = _qs(req, to_unicode=True)- self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})- self.assertEqual(set(fs[u'one']), {u'two', u'three'})- self.assertEqual(fs[u'test2'], [u'xxx µ'])- self.assertEqual(fs[u'six'], [u'seven'])+ self.assertEqual(set(fs['test £']), {'val1', 'val2'})+ self.assertEqual(set(fs['one']), {'two', 'three'})+ self.assertEqual(fs['test2'], ['xxx µ'])+ self.assertEqual(fs['six'], ['seven'])def test_from_response_post_nonascii_bytes_latin1(self):response = _buildresponse(@@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')self.assertEqual(req.url, "http://www.example.com/this/post.php")fs = _qs(req, to_unicode=True, encoding='latin1')- self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})- self.assertEqual(set(fs[u'one']), {u'two', u'three'})- self.assertEqual(fs[u'test2'], [u'xxx µ'])- self.assertEqual(fs[u'six'], [u'seven'])+ self.assertEqual(set(fs['test £']), {'val1', 'val2'})+ self.assertEqual(set(fs['one']), {'two', 'three'})+ self.assertEqual(fs['test2'], ['xxx µ'])+ self.assertEqual(fs['six'], ['seven'])def test_from_response_post_nonascii_unicode(self):response = _buildresponse(- u"""<form action="post.php" method="POST">+ """<form action="post.php" method="POST"><input type="hidden" name="test £" value="val1"><input type="hidden" name="test £" value="val2"><input type="hidden" name="test2" value="xxx µ">@@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')self.assertEqual(req.url, "http://www.example.com/this/post.php")fs = _qs(req, to_unicode=True)- self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})- self.assertEqual(set(fs[u'one']), {u'two', u'three'})- self.assertEqual(fs[u'test2'], [u'xxx µ'])- self.assertEqual(fs[u'six'], [u'seven'])+ self.assertEqual(set(fs['test £']), {'val1', 'val2'})+ self.assertEqual(set(fs['one']), {'two', 'three'})+ self.assertEqual(fs['test2'], ['xxx µ'])+ self.assertEqual(fs['six'], ['seven'])def test_from_response_duplicate_form_key(self):response = _buildresponse(@@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):<input type="hidden" name="two" value="clicked2"></form>""")req = self.request_class.from_response(- response, clickdata={u'name': u'clickable', u'value': u'clicked2'}+ response, clickdata={'name': 'clickable', 'value': 'clicked2'})fs = _qs(req)self.assertEqual(fs[b'clickable'], [b'clicked2'])@@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):def test_from_response_unicode_clickdata(self):response = _buildresponse(- u"""<form action="get.php" method="GET">+ """<form action="get.php" method="GET"><input type="submit" name="price in \u00a3" value="\u00a3 1000"><input type="submit" name="price in \u20ac" value="\u20ac 2000"><input type="hidden" name="poundsign" value="\u00a3"><input type="hidden" name="eurosign" value="\u20ac"></form>""")req = self.request_class.from_response(- response, clickdata={u'name': u'price in \u00a3'}+ response, clickdata={'name': 'price in \u00a3'})fs = _qs(req, to_unicode=True)- self.assertTrue(fs[u'price in \u00a3'])+ self.assertTrue(fs['price in \u00a3'])def test_from_response_unicode_clickdata_latin1(self):response = _buildresponse(- u"""<form action="get.php" method="GET">+ """<form action="get.php" method="GET"><input type="submit" name="price in \u00a3" value="\u00a3 1000"><input type="submit" name="price in \u00a5" value="\u00a5 2000"><input type="hidden" name="poundsign" value="\u00a3">@@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):</form>""",encoding='latin1')req = self.request_class.from_response(- response, clickdata={u'name': u'price in \u00a5'}+ response, clickdata={'name': 'price in \u00a5'})fs = _qs(req, to_unicode=True, encoding='latin1')- self.assertTrue(fs[u'price in \u00a5'])+ self.assertTrue(fs['price in \u00a5'])def test_from_response_multiple_forms_clickdata(self):response = _buildresponse(@@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):</form>""")req = self.request_class.from_response(- response, formname='form2', clickdata={u'name': u'clickable'}+ response, formname='form2', clickdata={'name': 'clickable'})fs = _qs(req)self.assertEqual(fs[b'clickable'], [b'clicked2'])@@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):def test_from_response_unicode_xpath(self):response = _buildresponse(b'<form name="\xd1\x8a"></form>')- r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")+ r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")fs = _qs(r)self.assertEqual(fs, {})- xpath = u"//form[@name='\u03b1']"+ xpath = "//form[@name='\u03b1']"self.assertRaisesRegex(ValueError, re.escape(xpath),self.request_class.from_response,response, formxpath=xpath)@@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):self._test_request(params=('value',))self._test_request(params=('username', 'password'), methodname='login')self._test_request(params=('response', ), methodresponse='login')- self._test_request(params=(u'pas£',), encoding='utf-8')+ self._test_request(params=('pas£',), encoding='utf-8')self._test_request(params=(None,), allow_none=1)self.assertRaises(TypeError, self._test_request)self.assertRaises(TypeError, self._test_request, params=(None,))def test_latin1(self):- self._test_request(params=(u'pas£',), encoding='latin1')+ self._test_request(params=('pas£',), encoding='latin1')class JsonRequestTest(RequestTest):diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex e0ca3c0e6..56d017de6 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):def test_unicode_url(self):# instantiate with unicode url without encoding (should set default encoding)- resp = self.response_class(u"http://www.example.com/")+ resp = self.response_class("http://www.example.com/")self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)# make sure urls are converted to str- resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')+ resp = self.response_class(url="http://www.example.com/", encoding='utf-8')assert isinstance(resp.url, str)- resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')+ resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))- resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')+ resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')- resp = self.response_class(u"http://www.example.com/price/\xa3",+ resp = self.response_class("http://www.example.com/price/\xa3",headers={"Content-type": ["text/html; charset=utf-8"]})self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))- resp = self.response_class(u"http://www.example.com/price/\xa3",+ resp = self.response_class("http://www.example.com/price/\xa3",headers={"Content-type": ["text/html; charset=iso-8859-1"]})self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')def test_unicode_body(self):unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ''\u0442\u0435\u043a\u0441\u0442')- self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')+ self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')original_string = unicode_string.encode('cp1251')r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')@@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):def test_encoding(self):r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",headers={"Content-type": ["text/html; charset=utf-8"]})- r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")+ r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")r3 = self.response_class("http://www.example.com", body=b"\xa3",headers={"Content-type": ["text/html; charset=iso-8859-1"]})r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")@@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):self.assertEqual(r5._headers_encoding(), None)self._assert_response_encoding(r5, "utf-8")assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'- self._assert_response_values(r1, 'utf-8', u"\xa3")- self._assert_response_values(r2, 'utf-8', u"\xa3")- self._assert_response_values(r3, 'iso-8859-1', u"\xa3")- self._assert_response_values(r6, 'gb18030', u"\u2015")- self._assert_response_values(r7, 'gb18030', u"\u2015")+ self._assert_response_values(r1, 'utf-8', "\xa3")+ self._assert_response_values(r2, 'utf-8', "\xa3")+ self._assert_response_values(r3, 'iso-8859-1', "\xa3")+ self._assert_response_values(r6, 'gb18030', "\u2015")+ self._assert_response_values(r7, 'gb18030', "\u2015")# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies- self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")+ self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")def test_declared_encoding_invalid(self):"""Check that unknown declared encodings are ignored"""@@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):headers={"Content-type": ["text/html; charset=UKNOWN"]},body=b"\xc2\xa3")self.assertEqual(r._declared_encoding(), None)- self._assert_response_values(r, 'utf-8', u"\xa3")+ self._assert_response_values(r, 'utf-8', "\xa3")def test_utf16(self):"""Test utf-16 because UnicodeDammit is known to have problems with"""r = self.response_class("http://www.example.com",body=b'\xff\xfeh\x00i\x00',encoding='utf-16')- self._assert_response_values(r, 'utf-16', u"hi")+ self._assert_response_values(r, 'utf-16', "hi")def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):r6 = self.response_class("http://www.example.com",@@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):body=b"\xef\xbb\xbfWORD\xe3\xab")self.assertEqual(r6.encoding, 'utf-8')self.assertIn(r6.text, {- u'WORD\ufffd\ufffd', # w3lib < 1.19.0- u'WORD\ufffd', # w3lib >= 1.19.0+ 'WORD\ufffd\ufffd', # w3lib < 1.19.0+ 'WORD\ufffd', # w3lib >= 1.19.0})def test_bom_is_removed_from_body(self):@@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):# Test response without content-type and BOM encodingresponse = self.response_class(url, body=body)self.assertEqual(response.encoding, 'utf-8')- self.assertEqual(response.text, u'WORD')+ self.assertEqual(response.text, 'WORD')response = self.response_class(url, body=body)- self.assertEqual(response.text, u'WORD')+ self.assertEqual(response.text, 'WORD')self.assertEqual(response.encoding, 'utf-8')# Body caching sideeffect isn't triggered when encoding is declared in@@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):# bodyresponse = self.response_class(url, headers=headers, body=body)self.assertEqual(response.encoding, 'utf-8')- self.assertEqual(response.text, u'WORD')+ self.assertEqual(response.text, 'WORD')response = self.response_class(url, headers=headers, body=body)- self.assertEqual(response.text, u'WORD')+ self.assertEqual(response.text, 'WORD')self.assertEqual(response.encoding, 'utf-8')def test_replace_wrong_encoding(self):"""Test invalid chars are replaced properly"""r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')# XXX: Policy for replacing invalid chars may suffer minor variations- # but it should always contain the unicode replacement char (u'\ufffd')- assert u'\ufffd' in r.text, repr(r.text)- assert u'PREFIX' in r.text, repr(r.text)- assert u'SUFFIX' in r.text, repr(r.text)+ # but it should always contain the unicode replacement char ('\ufffd')+ assert '\ufffd' in r.text, repr(r.text)+ assert 'PREFIX' in r.text, repr(r.text)+ assert 'SUFFIX' in r.text, repr(r.text)# Do not destroy html tags due to encoding bugsr = self.response_class("http://example.com", encoding='utf-8',body=b'\xf0<span>value</span>')- assert u'<span>value</span>' in r.text, repr(r.text)+ assert '<span>value</span>' in r.text, repr(r.text)# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')- # assert u'\ufffd' in r.text, repr(r.text)+ # assert '\ufffd' in r.text, repr(r.text)def test_selector(self):body = b"<html><head><title>Some page</title><body></body></html>"@@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):self.assertEqual(response.selector.xpath("//title/text()").getall(),- [u'Some page']+ ['Some page'])self.assertEqual(response.selector.css("title::text").getall(),- [u'Some page']+ ['Some page'])self.assertEqual(response.selector.re("Some (.*)</title>"),- [u'page']+ ['page'])def test_selector_shortcuts(self):@@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):resp1 = self.response_class('http://example.com',encoding='utf8',- body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8'))req = self._assert_followed_url(resp1.css('a')[0],@@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):resp2 = self.response_class('http://example.com',encoding='cp1251',- body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251'))req = self._assert_followed_url(resp2.css('a')[0],@@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):def test_body_as_unicode_deprecation_warning(self):with catch_warnings(record=True) as warnings:- r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')- self.assertEqual(r1.body_as_unicode(), u'Hello')+ r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')+ self.assertEqual(r1.body_as_unicode(), 'Hello')self.assertEqual(len(warnings), 1)self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)@@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):self.assertEqual(response.selector.xpath("//elem/text()").getall(),- [u'value']+ ['value'])def test_selector_shortcuts(self):diff --git a/tests/test_item.py b/tests/test_item.pyindex 60468971c..0ce78f8c0 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):name = Field()i = TestItem()- i['name'] = u'name'- self.assertEqual(i['name'], u'name')+ i['name'] = 'name'+ self.assertEqual(i['name'], 'name')def test_init(self):class TestItem(Item):@@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):i = TestItem()self.assertRaises(KeyError, i.__getitem__, 'name')- i2 = TestItem(name=u'john doe')- self.assertEqual(i2['name'], u'john doe')+ i2 = TestItem(name='john doe')+ self.assertEqual(i2['name'], 'john doe')- i3 = TestItem({'name': u'john doe'})- self.assertEqual(i3['name'], u'john doe')+ i3 = TestItem({'name': 'john doe'})+ self.assertEqual(i3['name'], 'john doe')i4 = TestItem(i3)- self.assertEqual(i4['name'], u'john doe')+ self.assertEqual(i4['name'], 'john doe')- self.assertRaises(KeyError, TestItem, {'name': u'john doe',- 'other': u'foo'})+ self.assertRaises(KeyError, TestItem, {'name': 'john doe',+ 'other': 'foo'})def test_invalid_field(self):class TestItem(Item):@@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):number = Field()i = TestItem()- i['name'] = u'John Doe'+ i['name'] = 'John Doe'i['number'] = 123itemrepr = repr(i)@@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):i = TestItem()self.assertRaises(KeyError, i.get_name)- i['name'] = u'lala'- self.assertEqual(i.get_name(), u'lala')- i.change_name(u'other')+ i['name'] = 'lala'+ self.assertEqual(i.get_name(), 'lala')+ i.change_name('other')self.assertEqual(i.get_name(), 'other')def test_metaclass(self):@@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):values = Field()i = TestItem()- i['name'] = u'John'+ i['name'] = 'John'self.assertEqual(list(i.keys()), ['name'])self.assertEqual(list(i.values()), ['John'])- i['keys'] = u'Keys'- i['values'] = u'Values'+ i['keys'] = 'Keys'+ i['values'] = 'Values'self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])- self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])+ self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])def test_metaclass_with_fields_attribute(self):class TestItem(Item):fields = {'new': Field(default='X')}- item = TestItem(new=u'New')+ item = TestItem(new='New')self.assertSortedEqual(list(item.keys()), ['new'])- self.assertSortedEqual(list(item.values()), [u'New'])+ self.assertSortedEqual(list(item.values()), ['New'])def test_metaclass_inheritance(self):class ParentItem(Item):@@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):name = Field()i = TestItem()- i['name'] = u'John'- self.assertEqual(dict(i), {'name': u'John'})+ i['name'] = 'John'+ self.assertEqual(dict(i), {'name': 'John'})def test_copy(self):class TestItem(Item):diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 8d4538eed..a0bafa5e5 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -31,31 +31,31 @@ class Base:page4_url = 'http://example.com/page%204.html'self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),- Link(url='http://www.google.com/something', text=u''),- Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url=page4_url, text=u'href with whitespaces'),+ Link(url='http://www.google.com/something', text=''),+ Link(url='http://example.com/innertag.html', text='inner tag'),+ Link(url=page4_url, text='href with whitespaces'),])def test_extract_filter_allow(self):lx = self.extractor_cls(allow=('sample', ))self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')])def test_extract_filter_allow_with_duplicates(self):lx = self.extractor_cls(allow=('sample', ), unique=False)self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')])@@ -63,10 +63,10 @@ class Base:lx = self.extractor_cls(allow=('sample', ), unique=False,canonicalize=True)self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')])@@ -74,22 +74,22 @@ class Base:lx = self.extractor_cls(allow=('sample',), unique=True,canonicalize=True)self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),])def test_extract_filter_allow_and_deny(self):lx = self.extractor_cls(allow=('sample', ), deny=('3', ))self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),])def test_extract_filter_allowed_domains(self):lx = self.extractor_cls(allow_domains=('google.com', ))self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://www.google.com/something', text=u''),+ Link(url='http://www.google.com/something', text=''),])def test_extraction_using_single_values(self):@@ -97,27 +97,27 @@ class Base:lx = self.extractor_cls(allow='sample')self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),Link(url='http://example.com/sample3.html#foo',text='sample 3 repetition with fragment')])lx = self.extractor_cls(allow='sample', deny='3')self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),])lx = self.extractor_cls(allow_domains='google.com')self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://www.google.com/something', text=u''),+ Link(url='http://www.google.com/something', text=''),])lx = self.extractor_cls(deny_domains='example.com')self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://www.google.com/something', text=u''),+ Link(url='http://www.google.com/something', text=''),])def test_nofollow(self):@@ -145,11 +145,11 @@ class Base:lx = self.extractor_cls()self.assertEqual(lx.extract_links(response), [- Link(url='http://example.org/about.html', text=u'About us'),- Link(url='http://example.org/follow.html', text=u'Follow this link'),- Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),- Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),+ Link(url='http://example.org/about.html', text='About us'),+ Link(url='http://example.org/follow.html', text='Follow this link'),+ Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),+ Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),])def test_matches(self):@@ -183,8 +183,8 @@ class Base:def test_restrict_xpaths(self):lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),])def test_restrict_xpaths_encoding(self):@@ -202,14 +202,14 @@ class Base:lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")self.assertEqual(lx.extract_links(response),- [Link(url='http://example.org/about.html', text=u'About us\xa3')])+ [Link(url='http://example.org/about.html', text='About us\xa3')])def test_restrict_xpaths_with_html_entities(self):html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)self.assertEqual(links,- [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])+ [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])def test_restrict_xpaths_concat_in_handle_data(self):"""html entities cause SGMLParser to call handle_data hook twice"""@@ -217,22 +217,22 @@ class Base:response = HtmlResponse("http://example.org", body=body, encoding='gb18030')lx = self.extractor_cls(restrict_xpaths="//div")self.assertEqual(lx.extract_links(response),- [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',+ [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',fragment='', nofollow=False)])def test_restrict_css(self):lx = self.extractor_cls(restrict_css=('#subwrapper a',))self.assertEqual(lx.extract_links(self.response), [- Link(url='http://example.com/sample2.html', text=u'sample 2')+ Link(url='http://example.com/sample2.html', text='sample 2')])def test_restrict_css_and_restrict_xpaths_together(self):lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),restrict_css=('#subwrapper + a', ))self.assertEqual([link for link in lx.extract_links(self.response)], [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),])def test_area_tag_with_unicode_present(self):@@ -243,7 +243,7 @@ class Base:lx.extract_links(response)lx.extract_links(response)self.assertEqual(lx.extract_links(response),- [Link(url='http://example.org/foo', text=u'',+ [Link(url='http://example.org/foo', text='',fragment='', nofollow=False)])def test_encoded_url(self):@@ -251,7 +251,7 @@ class Base:response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')lx = self.extractor_cls()self.assertEqual(lx.extract_links(response), [- Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),+ Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),])def test_encoded_url_in_restricted_xpath(self):@@ -259,7 +259,7 @@ class Base:response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')lx = self.extractor_cls(restrict_xpaths="//div")self.assertEqual(lx.extract_links(response), [- Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),+ Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),])def test_ignored_extensions(self):@@ -268,7 +268,7 @@ class Base:response = HtmlResponse("http://example.org/", body=html)lx = self.extractor_cls()self.assertEqual(lx.extract_links(response), [- Link(url='http://example.org/page.html', text=u'asd'),+ Link(url='http://example.org/page.html', text='asd'),])# override denied extensions@@ -308,25 +308,25 @@ class Base:page4_url = 'http://example.com/page%204.html'self.assertEqual(lx.extract_links(self.response), [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),- Link(url='http://www.google.com/something', text=u''),- Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url=page4_url, text=u'href with whitespaces'),+ Link(url='http://www.google.com/something', text=''),+ Link(url='http://example.com/innertag.html', text='inner tag'),+ Link(url=page4_url, text='href with whitespaces'),])lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())self.assertEqual(lx.extract_links(self.response), [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample2.jpg', text=u''),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample2.jpg', text=''),+ Link(url='http://example.com/sample3.html', text='sample 3 text'),Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),- Link(url='http://www.google.com/something', text=u''),- Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url=page4_url, text=u'href with whitespaces'),+ Link(url='http://www.google.com/something', text=''),+ Link(url='http://example.com/innertag.html', text='inner tag'),+ Link(url=page4_url, text='href with whitespaces'),])lx = self.extractor_cls(attrs=None)@@ -344,24 +344,24 @@ class Base:lx = self.extractor_cls()self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/sample1.html', text=u''),- Link(url='http://example.com/sample2.html', text=u'sample 2'),+ Link(url='http://example.com/sample1.html', text=''),+ Link(url='http://example.com/sample2.html', text='sample 2'),])lx = self.extractor_cls(tags="area")self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/sample1.html', text=u''),+ Link(url='http://example.com/sample1.html', text=''),])lx = self.extractor_cls(tags="a")self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/sample2.html', text=u'sample 2'),+ Link(url='http://example.com/sample2.html', text='sample 2'),])lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample2.jpg', text=u''),+ Link(url='http://example.com/sample2.html', text='sample 2'),+ Link(url='http://example.com/sample2.jpg', text=''),])def test_tags_attrs(self):@@ -375,14 +375,14 @@ class Base:lx = self.extractor_cls(tags='div', attrs='data-url')self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),- Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)+ Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),+ Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)])lx = self.extractor_cls(tags=('div',), attrs=('data-url',))self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),- Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)+ Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),+ Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)])def test_xhtml(self):@@ -420,13 +420,13 @@ class Base:self.assertEqual(lx.extract_links(response),[- Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),- Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',+ Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),+ Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),+ Link(url='http://example.com/nofollow.html', text='Dont follow this one',fragment='', nofollow=True),- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',+ Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',fragment='', nofollow=False),- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),])@@ -436,13 +436,13 @@ class Base:self.assertEqual(lx.extract_links(response),[- Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),- Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',+ Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),+ Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),+ Link(url='http://example.com/nofollow.html', text='Dont follow this one',fragment='', nofollow=True),- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',+ Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',fragment='', nofollow=False),- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),])@@ -455,8 +455,8 @@ class Base:response = HtmlResponse("http://example.org/index.html", body=html)lx = self.extractor_cls()self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),+ Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),+ Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),])def test_ftp_links(self):@@ -467,7 +467,7 @@ class Base:response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')lx = self.extractor_cls()self.assertEqual(lx.extract_links(response), [- Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),+ Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),])def test_pickle_extractor(self):@@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):response = HtmlResponse("http://example.org/index.html", body=html)lx = self.extractor_cls()self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),+ Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),+ Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),])def test_link_restrict_text(self):@@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):# Simple text inclusion testlx = self.extractor_cls(restrict_text='dog')self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),])# Unique regex testlx = self.extractor_cls(restrict_text=r'of.*dog')self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),])# Multiple regex testlx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),+ Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),])def test_restrict_xpaths_with_html_entities(self):diff --git a/tests/test_loader.py b/tests/test_loader.pyindex 581183625..2ed6f365f 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):def test_add_value_on_unknown_field(self):il = TestItemLoader()- self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])+ self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])def test_load_item_using_default_loader(self):i = TestItem()- i['summary'] = u'lala'+ i['summary'] = 'lala'il = ItemLoader(item=i)- il.add_value('name', u'marta')+ il.add_value('name', 'marta')item = il.load_item()assert item is i- self.assertEqual(item['summary'], [u'lala'])- self.assertEqual(item['name'], [u'marta'])+ self.assertEqual(item['summary'], ['lala'])+ self.assertEqual(item['name'], ['marta'])def test_load_item_using_custom_loader(self):il = TestItemLoader()- il.add_value('name', u'marta')+ il.add_value('name', 'marta')item = il.load_item()- self.assertEqual(item['name'], [u'Marta'])+ self.assertEqual(item['name'], ['Marta'])class InitializationTestMixin:@@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):self.assertRaises(RuntimeError, l.get_css, '#name::text')def test_init_method_with_selector(self):- sel = Selector(text=u"<html><body><div>marta</div></body></html>")+ sel = Selector(text="<html><body><div>marta</div></body></html>")l = TestItemLoader(selector=sel)self.assertIs(l.selector, sel)l.add_xpath('name', '//div/text()')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])def test_init_method_with_selector_css(self):- sel = Selector(text=u"<html><body><div>marta</div></body></html>")+ sel = Selector(text="<html><body><div>marta</div></body></html>")l = TestItemLoader(selector=sel)self.assertIs(l.selector, sel)l.add_css('name', 'div::text')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])def test_init_method_with_response(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_xpath('name', '//div/text()')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])def test_init_method_with_response_css(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_css('name', 'div::text')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.add_css('url', 'a::attr(href)')- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])# combining/accumulating CSS selectors and XPath expressionsl.add_xpath('name', '//div/text()')- self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])l.add_xpath('url', '//img/@src')- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])def test_add_xpath_re(self):l = TestItemLoader(response=self.response)l.add_xpath('name', '//div/text()', re='ma')- self.assertEqual(l.get_output_value('name'), [u'Ma'])+ self.assertEqual(l.get_output_value('name'), ['Ma'])def test_replace_xpath(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_xpath('name', '//div/text()')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.replace_xpath('name', '//p/text()')- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])l.replace_xpath('name', ['//p/text()', '//div/text()'])- self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])def test_get_xpath(self):l = TestItemLoader(response=self.response)- self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])- self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')- self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')+ self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])+ self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')+ self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')- self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])+ self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])def test_replace_xpath_multi_fields(self):l = TestItemLoader(response=self.response)l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])def test_replace_xpath_re(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_xpath('name', '//div/text()')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.replace_xpath('name', '//div/text()', re='ma')- self.assertEqual(l.get_output_value('name'), [u'Ma'])+ self.assertEqual(l.get_output_value('name'), ['Ma'])def test_add_css_re(self):l = TestItemLoader(response=self.response)l.add_css('name', 'div::text', re='ma')- self.assertEqual(l.get_output_value('name'), [u'Ma'])+ self.assertEqual(l.get_output_value('name'), ['Ma'])l.add_css('url', 'a::attr(href)', re='http://(.+)')- self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])def test_replace_css(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_css('name', 'div::text')- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.replace_css('name', 'p::text')- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])l.replace_css('name', ['p::text', 'div::text'])- self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])l.add_css('url', 'a::attr(href)', re='http://(.+)')- self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])l.replace_css('url', 'img::attr(src)')- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])def test_get_css(self):l = TestItemLoader(response=self.response)- self.assertEqual(l.get_css('p::text'), [u'paragraph'])- self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')- self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')+ self.assertEqual(l.get_css('p::text'), ['paragraph'])+ self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')+ self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')- self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])+ self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),- [u'http://www.scrapy.org', u'/images/logo.png'])+ ['http://www.scrapy.org', '/images/logo.png'])def test_replace_css_multi_fields(self):l = TestItemLoader(response=self.response)l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})- self.assertEqual(l.get_output_value('name'), [u'Marta'])+ self.assertEqual(l.get_output_value('name'), ['Marta'])l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])def test_replace_css_re(self):l = TestItemLoader(response=self.response)self.assertTrue(l.selector)l.add_css('url', 'a::attr(href)')- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')- self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['scrapy.org'])class SubselectorLoaderTest(unittest.TestCase):@@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):nl.add_css('name_div', '#id')nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())- self.assertEqual(l.get_output_value('name'), [u'marta'])- self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])- self.assertEqual(l.get_output_value('name_value'), [u'marta'])+ self.assertEqual(l.get_output_value('name'), ['marta'])+ self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])+ self.assertEqual(l.get_output_value('name_value'), ['marta'])self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))@@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):nl.add_css('name_div', '#id')nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())- self.assertEqual(l.get_output_value('name'), [u'marta'])- self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])- self.assertEqual(l.get_output_value('name_value'), [u'marta'])+ self.assertEqual(l.get_output_value('name'), ['marta'])+ self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])+ self.assertEqual(l.get_output_value('name_value'), ['marta'])self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))@@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):nl2 = nl1.nested_xpath('a')l.add_xpath('url', '//footer/a/@href')- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])nl1.replace_xpath('url', 'img/@src')- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])nl2.replace_xpath('url', '@href')- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])def test_nested_ordering(self):l = NestedItemLoader(response=self.response)@@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):l.add_xpath('url', '//footer/a/@href')self.assertEqual(l.get_output_value('url'), [- u'/images/logo.png',- u'http://www.scrapy.org',- u'homepage',- u'http://www.scrapy.org',+ '/images/logo.png',+ 'http://www.scrapy.org',+ 'homepage',+ 'http://www.scrapy.org',])def test_nested_load_item(self):@@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):assert item is nl1.itemassert item is nl2.item- self.assertEqual(item['name'], [u'marta'])- self.assertEqual(item['url'], [u'http://www.scrapy.org'])- self.assertEqual(item['image'], [u'/images/logo.png'])+ self.assertEqual(item['name'], ['marta'])+ self.assertEqual(item['url'], ['http://www.scrapy.org'])+ self.assertEqual(item['image'], ['/images/logo.png'])# Functions as processorsdiff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex d0a59e8cd..eb14de14f 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):def test_load_item_using_default_loader(self):i = TestItem()- i['summary'] = u'lala'+ i['summary'] = 'lala'il = ItemLoader(item=i)- il.add_value('name', u'marta')+ il.add_value('name', 'marta')item = il.load_item()assert item is i- self.assertEqual(item['summary'], [u'lala'])- self.assertEqual(item['name'], [u'marta'])+ self.assertEqual(item['summary'], ['lala'])+ self.assertEqual(item['name'], ['marta'])def test_load_item_using_custom_loader(self):il = TestItemLoader()- il.add_value('name', u'marta')+ il.add_value('name', 'marta')item = il.load_item()- self.assertEqual(item['name'], [u'Marta'])+ self.assertEqual(item['name'], ['Marta'])def test_load_item_ignore_none_field_values(self):def validate_sku(value):@@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):price_out = Compose(TakeFirst(), float)sku_out = Compose(TakeFirst(), validate_sku)- valid_fragment = u'SKU: 1234'- invalid_fragment = u'SKU: not available'+ valid_fragment = 'SKU: 1234'+ invalid_fragment = 'SKU: not available'sku_re = 'SKU: (.+)'il = MyLoader(item={})# Should not return "sku: None".il.add_value('sku', [invalid_fragment], re=sku_re)# Should not ignore empty values.- il.add_value('name', u'')- il.add_value('price', [u'0'])+ il.add_value('name', '')+ il.add_value('price', ['0'])self.assertEqual(il.load_item(), {- 'name': u'',+ 'name': '','price': 0.0,})il.replace_value('sku', [valid_fragment], re=sku_re)- self.assertEqual(il.load_item()['sku'], u'1234')+ self.assertEqual(il.load_item()['sku'], '1234')def test_self_referencing_loader(self):class MyLoader(ItemLoader):@@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):def test_add_value(self):il = TestItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_collected_values('name'), [u'Marta'])- self.assertEqual(il.get_output_value('name'), [u'Marta'])- il.add_value('name', u'pepe')- self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])- self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_collected_values('name'), ['Marta'])+ self.assertEqual(il.get_output_value('name'), ['Marta'])+ il.add_value('name', 'pepe')+ self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])+ self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])# test add object valueil.add_value('summary', {'key': 1})self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])- il.add_value(None, u'Jim', lambda x: {'name': x})- self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])+ il.add_value(None, 'Jim', lambda x: {'name': x})+ self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])def test_add_zero(self):il = NameItemLoader()@@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):def test_replace_value(self):il = TestItemLoader()- il.replace_value('name', u'marta')- self.assertEqual(il.get_collected_values('name'), [u'Marta'])- self.assertEqual(il.get_output_value('name'), [u'Marta'])- il.replace_value('name', u'pepe')- self.assertEqual(il.get_collected_values('name'), [u'Pepe'])- self.assertEqual(il.get_output_value('name'), [u'Pepe'])+ il.replace_value('name', 'marta')+ self.assertEqual(il.get_collected_values('name'), ['Marta'])+ self.assertEqual(il.get_output_value('name'), ['Marta'])+ il.replace_value('name', 'pepe')+ self.assertEqual(il.get_collected_values('name'), ['Pepe'])+ self.assertEqual(il.get_output_value('name'), ['Pepe'])- il.replace_value(None, u'Jim', lambda x: {'name': x})- self.assertEqual(il.get_collected_values('name'), [u'Jim'])+ il.replace_value(None, 'Jim', lambda x: {'name': x})+ self.assertEqual(il.get_collected_values('name'), ['Jim'])def test_get_value(self):il = NameItemLoader()- self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))- self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))- self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))+ self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))+ self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))+ self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))- il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')- self.assertEqual([u'foo'], il.get_collected_values('name'))- il.replace_value('name', u'name:bar', re=u'name:(.*)$')- self.assertEqual([u'bar'], il.get_collected_values('name'))+ il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')+ self.assertEqual(['foo'], il.get_collected_values('name'))+ il.replace_value('name', 'name:bar', re='name:(.*)$')+ self.assertEqual(['bar'], il.get_collected_values('name'))def test_iter_on_input_processor_input(self):class NameFirstItemLoader(NameItemLoader):name_in = TakeFirst()il = NameFirstItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_collected_values('name'), [u'marta'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_collected_values('name'), ['marta'])il = NameFirstItemLoader()- il.add_value('name', [u'marta', u'jose'])- self.assertEqual(il.get_collected_values('name'), [u'marta'])+ il.add_value('name', ['marta', 'jose'])+ self.assertEqual(il.get_collected_values('name'), ['marta'])il = NameFirstItemLoader()- il.replace_value('name', u'marta')- self.assertEqual(il.get_collected_values('name'), [u'marta'])+ il.replace_value('name', 'marta')+ self.assertEqual(il.get_collected_values('name'), ['marta'])il = NameFirstItemLoader()- il.replace_value('name', [u'marta', u'jose'])- self.assertEqual(il.get_collected_values('name'), [u'marta'])+ il.replace_value('name', ['marta', 'jose'])+ self.assertEqual(il.get_collected_values('name'), ['marta'])il = NameFirstItemLoader()- il.add_value('name', u'marta')- il.add_value('name', [u'jose', u'pedro'])- self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])+ il.add_value('name', 'marta')+ il.add_value('name', ['jose', 'pedro'])+ self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])def test_map_compose_filter(self):def filter_world(x):@@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])il = TestItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'Mart'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['Mart'])item = il.load_item()- self.assertEqual(item['name'], [u'Mart'])+ self.assertEqual(item['name'], ['Mart'])def test_default_input_processor(self):il = DefaultedItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'mart'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['mart'])def test_inherited_default_input_processor(self):class InheritDefaultedItemLoader(DefaultedItemLoader):passil = InheritDefaultedItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'mart'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['mart'])def test_input_processor_inheritance(self):class ChildItemLoader(TestItemLoader):url_in = MapCompose(lambda v: v.lower())il = ChildItemLoader()- il.add_value('url', u'HTTP://scrapy.ORG')- self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'Marta'])+ il.add_value('url', 'HTTP://scrapy.ORG')+ self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['Marta'])class ChildChildItemLoader(ChildItemLoader):url_in = MapCompose(lambda v: v.upper())summary_in = MapCompose(lambda v: v)il = ChildChildItemLoader()- il.add_value('url', u'http://scrapy.org')- self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'Marta'])+ il.add_value('url', 'http://scrapy.org')+ self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['Marta'])def test_empty_map_compose(self):class IdentityDefaultedItemLoader(DefaultedItemLoader):name_in = MapCompose()il = IdentityDefaultedItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'marta'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['marta'])def test_identity_input_processor(self):class IdentityDefaultedItemLoader(DefaultedItemLoader):name_in = Identity()il = IdentityDefaultedItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'marta'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['marta'])def test_extend_custom_input_processors(self):class ChildItemLoader(TestItemLoader):name_in = MapCompose(TestItemLoader.name_in, str.swapcase)il = ChildItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'mARTA'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['mARTA'])def test_extend_default_input_processors(self):class ChildDefaultedItemLoader(DefaultedItemLoader):name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)il = ChildDefaultedItemLoader()- il.add_value('name', u'marta')- self.assertEqual(il.get_output_value('name'), [u'MART'])+ il.add_value('name', 'marta')+ self.assertEqual(il.get_output_value('name'), ['MART'])def test_output_processor_using_function(self):il = TestItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])class TakeFirstItemLoader(TestItemLoader):- name_out = u" ".join+ name_out = " ".joinil = TakeFirstItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), u'Mar Ta')+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), 'Mar Ta')def test_output_processor_error(self):class TestItemLoader(ItemLoader):@@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):name_out = MapCompose(float)il = TestItemLoader()- il.add_value('name', [u'$10'])+ il.add_value('name', ['$10'])try:- float(u'$10')+ float('$10')except Exception as e:expected_exc_str = str(e)@@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):def test_output_processor_using_classes(self):il = TestItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])class TakeFirstItemLoader(TestItemLoader):name_out = Join()il = TakeFirstItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), u'Mar Ta')+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), 'Mar Ta')class TakeFirstItemLoader(TestItemLoader):name_out = Join("<br>")il = TakeFirstItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')def test_default_output_processor(self):il = TestItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])class LalaItemLoader(TestItemLoader):default_output_processor = Identity()il = LalaItemLoader()- il.add_value('name', [u'mar', u'ta'])- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])+ il.add_value('name', ['mar', 'ta'])+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])def test_loader_context_on_declaration(self):class ChildItemLoader(TestItemLoader):- url_in = MapCompose(processor_with_args, key=u'val')+ url_in = MapCompose(processor_with_args, key='val')il = ChildItemLoader()- il.add_value('url', u'text')+ il.add_value('url', 'text')self.assertEqual(il.get_output_value('url'), ['val'])- il.replace_value('url', u'text2')+ il.replace_value('url', 'text2')self.assertEqual(il.get_output_value('url'), ['val'])def test_loader_context_on_instantiation(self):class ChildItemLoader(TestItemLoader):url_in = MapCompose(processor_with_args)- il = ChildItemLoader(key=u'val')- il.add_value('url', u'text')+ il = ChildItemLoader(key='val')+ il.add_value('url', 'text')self.assertEqual(il.get_output_value('url'), ['val'])- il.replace_value('url', u'text2')+ il.replace_value('url', 'text2')self.assertEqual(il.get_output_value('url'), ['val'])def test_loader_context_on_assign(self):@@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):url_in = MapCompose(processor_with_args)il = ChildItemLoader()- il.context['key'] = u'val'- il.add_value('url', u'text')+ il.context['key'] = 'val'+ il.add_value('url', 'text')self.assertEqual(il.get_output_value('url'), ['val'])- il.replace_value('url', u'text2')+ il.replace_value('url', 'text2')self.assertEqual(il.get_output_value('url'), ['val'])def test_item_passed_to_input_processor_functions(self):@@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):it = TestItem(name='marta')il = ChildItemLoader(item=it)- il.add_value('url', u'text')+ il.add_value('url', 'text')self.assertEqual(il.get_output_value('url'), ['marta'])- il.replace_value('url', u'text2')+ il.replace_value('url', 'text2')self.assertEqual(il.get_output_value('url'), ['marta'])def test_compose_processor(self):@@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])il = TestItemLoader()- il.add_value('name', [u'marta', u'other'])- self.assertEqual(il.get_output_value('name'), u'Mart')+ il.add_value('name', ['marta', 'other'])+ self.assertEqual(il.get_output_value('name'), 'Mart')item = il.load_item()- self.assertEqual(item['name'], u'Mart')+ self.assertEqual(item['name'], 'Mart')def test_partial_processor(self):def join(values, sep=None, loader_context=None, ignored=None):@@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):summary_out = Compose(partial(join, ignored='foo'))il = TestItemLoader()- il.add_value('name', [u'rabbit', u'hole'])- il.add_value('url', [u'rabbit', u'hole'])- il.add_value('summary', [u'rabbit', u'hole'])+ il.add_value('name', ['rabbit', 'hole'])+ il.add_value('url', ['rabbit', 'hole'])+ il.add_value('summary', ['rabbit', 'hole'])item = il.load_item()- self.assertEqual(item['name'], u'rabbit+hole')- self.assertEqual(item['url'], u'rabbit.hole')- self.assertEqual(item['summary'], u'rabbithole')+ self.assertEqual(item['name'], 'rabbit+hole')+ self.assertEqual(item['url'], 'rabbit.hole')+ self.assertEqual(item['summary'], 'rabbithole')def test_error_input_processor(self):class TestItem(Item):@@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):il = TestItemLoader()self.assertRaises(ValueError, il.add_value, 'name',- [u'marta', u'other'])+ ['marta', 'other'])def test_error_output_processor(self):class TestItem(Item):@@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):name_out = Compose(Join(), float)il = TestItemLoader()- il.add_value('name', u'marta')+ il.add_value('name', 'marta')with self.assertRaises(ValueError):il.load_item()@@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):il = TestItemLoader()self.assertRaises(ValueError, il.add_value, 'name',- [u'marta', u'other'], Compose(float))+ ['marta', 'other'], Compose(float))class InitializationFromDictTest(unittest.TestCase):@@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):def test_join(self):proc = Join()self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])- self.assertEqual(proc(['', 'hello', 'world']), u' hello world')- self.assertEqual(proc(['hello', 'world']), u'hello world')+ self.assertEqual(proc(['', 'hello', 'world']), ' hello world')+ self.assertEqual(proc(['hello', 'world']), 'hello world')self.assertIsInstance(proc(['hello', 'world']), str)def test_compose(self):@@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):def filter_world(x):return None if x == 'world' else xproc = MapCompose(filter_world, str.upper)- self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),- [u'HELLO', u'THIS', u'IS', u'SCRAPY'])+ self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),+ ['HELLO', 'THIS', 'IS', 'SCRAPY'])proc = MapCompose(filter_world, str.upper)self.assertEqual(proc(None), [])proc = MapCompose(filter_world, str.upper)diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 7064337ad..b771e7d79 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):def test_dropped(self):item = {}- exception = Exception(u"\u2018")+ exception = Exception("\u2018")response = Response("http://www.example.com")logkws = self.formatter.dropped(item, exception, response, self.spider)logline = logkws['msg'] % logkws['args']lines = logline.splitlines()assert all(isinstance(x, str) for x in lines)- self.assertEqual(lines, [u"Dropped: \u2018", '{}'])+ self.assertEqual(lines, ["Dropped: \u2018", '{}'])def test_item_error(self):# In practice, the complete traceback is shown by passing the@@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):response = Response("http://www.example.com")logkws = self.formatter.item_error(item, exception, response, self.spider)logline = logkws['msg'] % logkws['args']- self.assertEqual(logline, u"Error processing {'key': 'value'}")+ self.assertEqual(logline, "Error processing {'key': 'value'}")def test_spider_error(self):# In practice, the complete traceback is shown by passing the@@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):def test_scraped(self):item = CustomItem()- item['name'] = u'\xa3'+ item['name'] = '\xa3'response = Response("http://www.example.com")logkws = self.formatter.scraped(item, response, self.spider)logline = logkws['msg'] % logkws['args']lines = logline.splitlines()assert all(isinstance(x, str) for x in lines)- self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])+ self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])class LogFormatterSubclass(LogFormatter):def crawled(self, request, response, spider):kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)CRAWLEDMSG = (- u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"+ "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s")log_args = kwargs['args']log_args['flags'] = str(request.flags)diff --git a/tests/test_mail.py b/tests/test_mail.pyindex 53dbc0686..9b248fbfa 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):self.catched_msg = dict(**kwargs)def test_send_utf8(self):- subject = u'sübjèçt'- body = u'bödÿ-àéïöñß'+ subject = 'sübjèçt'+ body = 'bödÿ-àéïöñß'mailsender = MailSender(debug=True)mailsender.send(to=['[email protected]'], subject=subject, body=body,charset='utf-8', _callback=self._catch_mail_sent)@@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')def test_send_attach_utf8(self):- subject = u'sübjèçt'- body = u'bödÿ-àéïöñß'+ subject = 'sübjèçt'+ body = 'bödÿ-àéïöñß'attach = BytesIO()attach.write(body.encode('utf-8'))attach.seek(0)diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex dd19a69d5..a175f88ca 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):mappings = [(b'attachment; filename="data.xml"', XmlResponse),(b'attachment; filename=data.xml', XmlResponse),- (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),- (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),- (u'attachment;filename=data高.doc'.encode('gbk'), Response),- (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),- (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),+ ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),+ ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),+ ('attachment;filename=data高.doc'.encode('gbk'), Response),+ ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),+ ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),]for source, cls in mappings:diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 24aaaf7ec..9d8c201dd 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -93,7 +93,7 @@ class BaseRobotParserTest:self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))def test_unicode_url_and_useragent(self):- robotstxt_robotstxt_body = u"""+ robotstxt_robotstxt_body = """User-Agent: *Disallow: /admin/Disallow: /static/@@ -107,11 +107,11 @@ class BaseRobotParserTest:self.assertTrue(rp.allowed("https://site.local/", "*"))self.assertFalse(rp.allowed("https://site.local/admin/", "*"))self.assertFalse(rp.allowed("https://site.local/static/", "*"))- self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))+ self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))- self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))+ self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))- self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))+ self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 00e663c11..62036ad8c 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):)self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],- [u'a']+ ['a'])self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],- [u'12.0']+ ['12.0'])self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),- [u'xpathrules']+ ['xpathrules'])self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],- [u'12']+ ['12'])def test_root_base_url(self):@@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))self.assertEqual(sel.type, 'xml')self.assertEqual(sel.xpath("//div").getall(),- [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])+ ['<div><img src="a.jpg"><p>Hello</p></img></div>'])sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))self.assertEqual(sel.type, 'html')self.assertEqual(sel.xpath("//div").getall(),- [u'<div><img src="a.jpg"><p>Hello</p></div>'])+ ['<div><img src="a.jpg"><p>Hello</p></div>'])def test_http_header_encoding_precedence(self):- # u'\xa3' = pound symbol in unicode- # u'\xc2\xa3' = pound symbol in utf-8- # u'\xa3' = pound symbol in latin-1 (iso-8859-1)-- meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'- head = u'<head>' + meta + u'</head>'- body_content = u'<span id="blank">\xa3</span>'- body = u'<body>' + body_content + u'</body>'- html = u'<html>' + head + body + u'</html>'+ # '\xa3' = pound symbol in unicode+ # '\xc2\xa3' = pound symbol in utf-8+ # '\xa3' = pound symbol in latin-1 (iso-8859-1)++ meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'+ head = '<head>' + meta + '</head>'+ body_content = '<span id="blank">\xa3</span>'+ body = '<body>' + body_content + '</body>'+ html = '<html>' + head + body + '</html>'encoding = 'utf-8'html_utf8 = html.encode(encoding)headers = {'Content-Type': ['text/html; charset=utf-8']}response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)x = Selector(response)- self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])def test_badly_encoded_body(self):# \xe9 alone isn't valid utf8 sequence@@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):def test_selector_bad_args(self):with self.assertRaisesRegex(ValueError, 'received both response and text'):- Selector(TextResponse(url='http://example.com', body=b''), text=u'')+ Selector(TextResponse(url='http://example.com', body=b''), text='')diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 83c10a3c3..78157a9b9 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):output = list(spider._parse(response))self.assertEqual(len(output), 2, iterator)self.assertEqual(output, [- {'loc': [u'http://www.example.com/Special-Offers.html'],- 'updated': [u'2009-08-16'],- 'custom': [u'fuu'],- 'other': [u'bar']},+ {'loc': ['http://www.example.com/Special-Offers.html'],+ 'updated': ['2009-08-16'],+ 'custom': ['fuu'],+ 'other': ['bar']},{'loc': [],- 'updated': [u'2009-08-16'],- 'other': [u'foo'],+ 'updated': ['2009-08-16'],+ 'other': ['foo'],'custom': []},], iterator)diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex d17bb2cbc..298178f08 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):def test_xmliter_unicode(self):# example taken from https://github.com/scrapy/scrapy/issues/1665- body = u"""<?xml version="1.0" encoding="UTF-8"?>+ body = """<?xml version="1.0" encoding="UTF-8"?><þingflokkar><þingflokkur id="26"><heiti />@@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):XmlResponse(url="http://example.com", body=body, encoding='utf-8'),):attrs = []- for x in self.xmliter(r, u'þingflokkur'):+ for x in self.xmliter(r, 'þingflokkur'):attrs.append((x.attrib['id'],- x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),- x.xpath(u'./tímabil/fyrstaþing/text()').getall()))+ x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),+ x.xpath('./tímabil/fyrstaþing/text()').getall()))self.assertEqual(attrs,- [(u'26', [u'-'], [u'80']),- (u'21', [u'Ab'], [u'76']),- (u'27', [u'A'], [u'27'])])+ [('26', ['-'], ['80']),+ ('21', ['Ab'], ['76']),+ ('27', ['A'], ['27'])])def test_xmliter_text(self):body = (@@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):)self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],- [[u'one'], [u'two']])+ [['one'], ['two']])def test_xmliter_namespaces(self):body = b"""@@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):response = XmlResponse('http://www.example.com', body=body)self.assertEqual(next(self.xmliter(response, 'item')).get(),- u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'+ '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>')@@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):result = [row for row in csv]self.assertEqual(result,- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])# explicit type check cuz' we no like stinkin' autocasting! yarrrfor result_row in result:@@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):csv = csviter(response, delimiter='\t')self.assertEqual([row for row in csv],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])def test_csviter_quotechar(self):body1 = get_testdata('feeds', 'feed-sample6.csv')@@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):csv1 = csviter(response1, quotechar="'")self.assertEqual([row for row in csv1],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])response2 = TextResponse(url="http://example.com/", body=body2)csv2 = csviter(response2, delimiter="|", quotechar="'")self.assertEqual([row for row in csv2],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])def test_csviter_wrong_quotechar(self):body = get_testdata('feeds', 'feed-sample6.csv')@@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):csv = csviter(response)self.assertEqual([row for row in csv],- [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},- {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},- {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},- {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])+ [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},+ {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},+ {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},+ {"'id'": "4", "'name'": "'empty'", "'value'": ""}])def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')@@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):csv = csviter(response, delimiter='\t')self.assertEqual([row for row in csv],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])def test_csviter_headers(self):sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()@@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):csv = csviter(response, headers=[h.decode('utf-8') for h in headers])self.assertEqual([row for row in csv],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},+ {'id': '4', 'name': 'empty', 'value': ''}])def test_csviter_falserow(self):body = get_testdata('feeds', 'feed-sample3.csv')@@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):csv = csviter(response)self.assertEqual([row for row in csv],- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},- {u'id': u'4', u'name': u'empty', u'value': u''}])+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},+ {'id': '4', 'name': 'empty', 'value': ''}])def test_csviter_exception(self):body = get_testdata('feeds', 'feed-sample3.csv')@@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):self.assertEqual(list(csv),[- {u'id': u'1', u'name': u'latin1', u'value': u'test'},- {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},+ {'id': '1', 'name': 'latin1', 'value': 'test'},+ {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},])@@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):self.assertEqual(list(csv),[- {u'id': u'1', u'name': u'cp852', u'value': u'test'},- {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},+ {'id': '1', 'name': 'cp852', 'value': 'test'},+ {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},])diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex ebce3c079..3f93f509e 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):class ToUnicodeTest(unittest.TestCase):def test_converting_an_utf8_encoded_string_to_unicode(self):- self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')+ self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')def test_converting_a_latin_1_encoded_string_to_unicode(self):- self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')+ self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):- self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')+ self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')def test_converting_a_strange_object_should_raise_TypeError(self):self.assertRaises(TypeError, to_unicode, 423)@@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):def test_errors_argument(self):self.assertEqual(to_unicode(b'a\xedb', 'utf-8', errors='replace'),- u'a\ufffdb'+ 'a\ufffdb')class ToBytesTest(unittest.TestCase):def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):- self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')+ self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):- self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')+ self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')@@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):def test_errors_argument(self):self.assertEqual(- to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),+ to_bytes('a\ufffdb', 'latin-1', errors='replace'),b'a?b')@@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):assert binary_is_text(b"hello")def test_utf_16_strings_contain_null_bytes(self):- assert binary_is_text(u"hello".encode('utf-16'))+ assert binary_is_text("hello".encode('utf-16'))def test_one_with_encoding(self):assert binary_is_text(b"<div>Price \xa3</div>")diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.pyindex 450e4bdca..de94ec960 100644--- a/tests/test_utils_reqser.py+++ b/tests/test_utils_reqser.py@@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):method="POST",body=b"some body",headers={'content-encoding': 'text/html; charset=latin-1'},- cookies={'currency': u'руб'},+ cookies={'currency': 'руб'},encoding='latin-1',priority=20,meta={'a': 'b'},diff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 5a52dd695..5ff2e41ef 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):def test_simple_render(self):context = dict(project_name='proj', name='spi', classname='TheSpider')- template = u'from ${project_name}.spiders.${name} import ${classname}'- rendered = u'from proj.spiders.spi import TheSpider'+ template = 'from ${project_name}.spiders.${name} import ${classname}'+ rendered = 'from proj.spiders.spi import TheSpider'template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')render_path = os.path.join(self.tmp_path, 'templ.py')