Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py

Konstantin Lopuhin47d3c63338c4
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index e61eedf50..3888da210 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.exceptions import UsageError
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.python import to_unicode
class Command(ScrapyCommand):
@@ -30,7 +31,8 @@ class Command(ScrapyCommand):
def _print_headers(self, headers, prefix):
for key, values in headers.items():
for value in values:
- print('%s %s: %s' % (prefix, key, value))
+ print('%s %s: %s' % (
+ prefix, to_unicode(key), to_unicode(value)))
def _print_response(self, response, opts):
if opts.headers:
@@ -38,7 +40,7 @@ class Command(ScrapyCommand):
print('>')
self._print_headers(response.headers, '<')
else:
- print(response.body)
+ print(to_unicode(response.body))
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 01508bdb4..ad0375443 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -22,13 +22,13 @@ class SiteTest(object):
def test_site():
r = resource.Resource()
- r.putChild("text", static.Data("Works", "text/plain"))
- r.putChild("html", static.Data("<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
- r.putChild("enc-gb18030", static.Data("<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
- r.putChild("redirect", util.Redirect("/redirected"))
- r.putChild("redirected", static.Data("Redirected here", "text/plain"))
+ r.putChild(b"text", static.Data(b"Works", "text/plain"))
+ r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
+ r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
+ r.putChild(b"redirect", util.Redirect(b"/redirected"))
+ r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
return server.Site(r)
-
+
if __name__ == '__main__':
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 1f7d85ef7..1f0f34c49 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_closespider.py
-tests/test_command_fetch.py
-tests/test_command_shell.py
tests/test_exporters.py
tests/test_linkextractors_deprecated.py
tests/test_crawl.py
@@ -12,7 +9,6 @@ tests/test_pipeline_files.py
tests/test_pipeline_images.py
tests/test_proxy_connect.py
tests/test_spidermiddleware_httperror.py
-tests/test_utils_template.py
scrapy/xlib/tx/iweb.py
scrapy/xlib/tx/interfaces.py
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 5283852b7..4843a9a2f 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_output(self):
_, out, _ = yield self.execute([self.url('/text')])
- self.assertEqual(out.strip(), 'Works')
+ self.assertEqual(out.strip(), b'Works')
@defer.inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
- out = out.replace('\r', '') # required on win32
- assert 'Server: TwistedWeb' in out
- assert 'Content-Type: text/plain' in out
+ out = out.replace(b'\r', b'') # required on win32
+ assert b'Server: TwistedWeb' in out, out
+ assert b'Content-Type: text/plain' in out
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index a56236d54..105202754 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_empty(self):
_, out, _ = yield self.execute(['-c', 'item'])
- assert '{}' in out
+ assert b'{}' in out
@defer.inlineCallbacks
def test_response_body(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
- assert 'Works' in out
+ assert b'Works' in out
@defer.inlineCallbacks
def test_response_type_text(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
- assert 'TextResponse' in out
+ assert b'TextResponse' in out
@defer.inlineCallbacks
def test_response_type_html(self):
_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
- assert 'HtmlResponse' in out
+ assert b'HtmlResponse' in out
@defer.inlineCallbacks
def test_response_selector_html(self):
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
- self.assertEqual(out.strip(), 'Works')
+ self.assertEqual(out.strip(), b'Works')
@defer.inlineCallbacks
def test_response_encoding_gb18030(self):
_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
- self.assertEqual(out.strip(), 'gb18030')
+ self.assertEqual(out.strip(), b'gb18030')
@defer.inlineCallbacks
def test_redirect(self):
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
- assert out.strip().endswith('/redirected')
+ assert out.strip().endswith(b'/redirected')
@defer.inlineCallbacks
def test_request_replace(self):