Scrapy, a fast high-level web crawling & scraping framework for Python.
Add test for already failed deferreds when downloading page in robots.txt middleware.
Artur Gaspardc8701ea429d
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 5f45dcb82..f2e94e171 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))return deferred+ def test_robotstxt_immediate_error(self):+ self.crawler.settings.set('ROBOTSTXT_OBEY', True)+ err = error.DNSLookupError('Robotstxt address not found')+ def immediate_failure(request, spider):+ deferred = Deferred()+ deferred.errback(failure.Failure(err))+ return deferred+ self.crawler.engine.download.side_effect = immediate_failure++ middleware = RobotsTxtMiddleware(self.crawler)+ return self.assertNotIgnored(Request('http://site.local'), middleware)+def test_ignore_robotstxt_request(self):self.crawler.settings.set('ROBOTSTXT_OBEY', True)def ignore_request(request, spider):