Scrapy, a fast high-level web crawling & scraping framework for Python.
Update tests for max_retry_times
harshasrinivas0d9ebd6e1ed5
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 064c740c9..5f1760fd1 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -103,29 +103,75 @@ class RetryTest(unittest.TestCase):req = self.mw.process_exception(req, exception, self.spider)self.assertEqual(req, None)- def test_different_retry(self):++class MaxRetryTimesTest(unittest.TestCase):+ def setUp(self):+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider('foo')+ self.mw = RetryMiddleware.from_crawler(self.crawler)+ self.mw.max_retry_times = 2++ def test_without_metakey(self):- req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})+ req = Request('http://www.scrapytest.org/invalid_url')- # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2- self._test_retry(req, DNSLookupError('foo'), 3)+ # SETTINGS: RETRY_TIMES is NON-ZERO+ self.mw.max_retry_times = 5+ self._test_retry(req, DNSLookupError('foo'), 5)- req2 = Request('http://www.scrapytest.org/invalid_url')+ # SETTINGS: RETRY_TIMES = 0+ self.mw.max_retry_times = 0+ self._test_retry(req, DNSLookupError('foo'), 0)++ # RESET RETRY_TIMES SETTINGS+ self.mw.max_retry_times = 2++ def test_with_metakey_preceding(self):+ # request with meta(max_retry_times) is called first+ req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})+ req2 = Request('http://www.scrapytest.org/invalid_url')+ req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})+# SETINGS: RETRY_TIMES < meta(max_retry_times)+ self.mw.max_retry_times = 2+ self._test_retry(req1, DNSLookupError('foo'), 3)self._test_retry(req2, DNSLookupError('foo'), 2)- # SETINGS: RETRY_TIMES = 0- self.mw.max_retry_times = 0- self._test_retry(req2, DNSLookupError('foo'), 0)+ # SETINGS: RETRY_TIMES > meta(max_retry_times)+ self.mw.max_retry_times = 5+ self._test_retry(req3, DNSLookupError('foo'), 4)+ self._test_retry(req2, DNSLookupError('foo'), 5)++ # RESET RETRY_TIMES SETTINGS+ self.mw.max_retry_times = 2++ def test_with_metakey_succeeding(self):+ # request with meta(max_retry_times) is called second+ req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})+ req2 = Request('http://www.scrapytest.org/invalid_url')+ req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})++ # SETINGS: RETRY_TIMES < meta(max_retry_times)+ self.mw.max_retry_times = 2+ self._test_retry(req2, DNSLookupError('foo'), 2)+ self._test_retry(req1, DNSLookupError('foo'), 3)+# SETINGS: RETRY_TIMES > meta(max_retry_times)- self.mw.max_retry_times = 4- self._test_retry(req2, DNSLookupError('foo'), 4)+ self.mw.max_retry_times = 5+ self._test_retry(req2, DNSLookupError('foo'), 5)+ self._test_retry(req3, DNSLookupError('foo'), 4)# RESET RETRY_TIMES SETTINGSself.mw.max_retry_times = 2+ def test_with_metakey_zero(self):++ req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 0})+ self._test_retry(req, DNSLookupError('foo'), 0)++def _test_retry(self, req, exception, max_retry_times):while max_retry_times > 0: