Scrapy, a fast high-level web crawling & scraping framework for Python.
py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py
Konstantin Lopuhin47d3c63338c4
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex e61eedf50..3888da210 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommandfrom scrapy.http import Requestfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.spider import spidercls_for_request, DefaultSpider+from scrapy.utils.python import to_unicodeclass Command(ScrapyCommand):@@ -30,7 +31,8 @@ class Command(ScrapyCommand):def _print_headers(self, headers, prefix):for key, values in headers.items():for value in values:- print('%s %s: %s' % (prefix, key, value))+ print('%s %s: %s' % (+ prefix, to_unicode(key), to_unicode(value)))def _print_response(self, response, opts):if opts.headers:@@ -38,7 +40,7 @@ class Command(ScrapyCommand):print('>')self._print_headers(response.headers, '<')else:- print(response.body)+ print(to_unicode(response.body))def run(self, args, opts):if len(args) != 1 or not is_url(args[0]):diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex 01508bdb4..ad0375443 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -22,13 +22,13 @@ class SiteTest(object):def test_site():r = resource.Resource()- r.putChild("text", static.Data("Works", "text/plain"))- r.putChild("html", static.Data("<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))- r.putChild("enc-gb18030", static.Data("<p>gb18030 encoding</p>", "text/html; charset=gb18030"))- r.putChild("redirect", util.Redirect("/redirected"))- r.putChild("redirected", static.Data("Redirected here", "text/plain"))+ r.putChild(b"text", static.Data(b"Works", "text/plain"))+ r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))+ r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))+ r.putChild(b"redirect", util.Redirect(b"/redirected"))+ r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))return server.Site(r)-+if __name__ == '__main__':port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txtindex 1f7d85ef7..1f0f34c49 100644--- a/tests/py3-ignores.txt+++ b/tests/py3-ignores.txt@@ -1,6 +1,3 @@-tests/test_closespider.py-tests/test_command_fetch.py-tests/test_command_shell.pytests/test_exporters.pytests/test_linkextractors_deprecated.pytests/test_crawl.py@@ -12,7 +9,6 @@ tests/test_pipeline_files.pytests/test_pipeline_images.pytests/test_proxy_connect.pytests/test_spidermiddleware_httperror.py-tests/test_utils_template.pyscrapy/xlib/tx/iweb.pyscrapy/xlib/tx/interfaces.pydiff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex 5283852b7..4843a9a2f 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_output(self):_, out, _ = yield self.execute([self.url('/text')])- self.assertEqual(out.strip(), 'Works')+ self.assertEqual(out.strip(), b'Works')@defer.inlineCallbacksdef test_headers(self):_, out, _ = yield self.execute([self.url('/text'), '--headers'])- out = out.replace('\r', '') # required on win32- assert 'Server: TwistedWeb' in out- assert 'Content-Type: text/plain' in out+ out = out.replace(b'\r', b'') # required on win32+ assert b'Server: TwistedWeb' in out, out+ assert b'Content-Type: text/plain' in outdiff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex a56236d54..105202754 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_empty(self):_, out, _ = yield self.execute(['-c', 'item'])- assert '{}' in out+ assert b'{}' in out@defer.inlineCallbacksdef test_response_body(self):_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])- assert 'Works' in out+ assert b'Works' in out@defer.inlineCallbacksdef test_response_type_text(self):_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])- assert 'TextResponse' in out+ assert b'TextResponse' in out@defer.inlineCallbacksdef test_response_type_html(self):_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])- assert 'HtmlResponse' in out+ assert b'HtmlResponse' in out@defer.inlineCallbacksdef test_response_selector_html(self):xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])- self.assertEqual(out.strip(), 'Works')+ self.assertEqual(out.strip(), b'Works')@defer.inlineCallbacksdef test_response_encoding_gb18030(self):_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])- self.assertEqual(out.strip(), 'gb18030')+ self.assertEqual(out.strip(), b'gb18030')@defer.inlineCallbacksdef test_redirect(self):_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])- assert out.strip().endswith('/redirected')+ assert out.strip().endswith(b'/redirected')@defer.inlineCallbacksdef test_request_replace(self):