Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add test for fetch command with redirections disabled

Paul Tremberth9aefc0a886ca
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index ad0375443..e50a989b3 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -20,12 +20,20 @@ class SiteTest(object):
return urljoin(self.baseurl, path)
+class NoMetaRefreshRedirect(util.Redirect):
+ def render(self, request):
+ content = util.Redirect.render(self, request)
+ return content.replace(b'http-equiv=\"refresh\"',
+ b'http-no-equiv=\"do-not-refresh-me\"')
+
+
def test_site():
r = resource.Resource()
r.putChild(b"text", static.Data(b"Works", "text/plain"))
r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
r.putChild(b"redirect", util.Redirect(b"/redirected"))
+ r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
return server.Site(r)
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 4843a9a2f..45d03a129 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
_, out, _ = yield self.execute([self.url('/text')])
self.assertEqual(out.strip(), b'Works')
+ @defer.inlineCallbacks
+ def test_redirect_default(self):
+ _, out, _ = yield self.execute([self.url('/redirect')])
+ self.assertEqual(out.strip(), b'Redirected here')
+
+ @defer.inlineCallbacks
+ def test_redirect_disabled(self):
+ _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')])
+ err = err.strip()
+ self.assertIn(b'downloader/response_status_count/302', err, err)
+ self.assertNotIn(b'downloader/response_status_count/200', err, err)
+
@defer.inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url('/text'), '--headers'])