Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add test for already failed deferreds when downloading page in robots.txt middleware.

Artur Gaspardc8701ea429d
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 5f45dcb82..f2e94e171 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
return deferred
+ def test_robotstxt_immediate_error(self):
+ self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+ err = error.DNSLookupError('Robotstxt address not found')
+ def immediate_failure(request, spider):
+ deferred = Deferred()
+ deferred.errback(failure.Failure(err))
+ return deferred
+ self.crawler.engine.download.side_effect = immediate_failure
+
+ middleware = RobotsTxtMiddleware(self.crawler)
+ return self.assertNotIgnored(Request('http://site.local'), middleware)
+
def test_ignore_robotstxt_request(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
def ignore_request(request, spider):