Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update unittests for max_retry_times

harshasrinivase321ac993100
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index cc3d37075..064c740c9 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -105,27 +105,34 @@ class RetryTest(unittest.TestCase):
def test_different_retry(self):
- req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1})
- self._test_retry(req, DNSLookupError('foo'))
- req2 = Request('http://www.scrapytest.org/invalid_url')
- self._test_retry(req2, DNSLookupError('foo'))
+ req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
- stats = self.crawler.stats
- assert stats.get_value('retry/max_reached') == 2
- assert stats.get_value('retry/count') == 3
+ # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2
+ self._test_retry(req, DNSLookupError('foo'), 3)
- def _test_retry(self, req, exception):
+ req2 = Request('http://www.scrapytest.org/invalid_url')
- req = self.mw.process_exception(req, exception, self.spider)
- assert isinstance(req, Request)
+ # SETINGS: RETRY_TIMES < meta(max_retry_times)
+ self._test_retry(req2, DNSLookupError('foo'), 2)
- retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times
+ # SETINGS: RETRY_TIMES = 0
+ self.mw.max_retry_times = 0
+ self._test_retry(req2, DNSLookupError('foo'), 0)
+
+ # SETINGS: RETRY_TIMES > meta(max_retry_times)
+ self.mw.max_retry_times = 4
+ self._test_retry(req2, DNSLookupError('foo'), 4)
+
+ # RESET RETRY_TIMES SETTINGS
+ self.mw.max_retry_times = 2
- while req.meta['retry_times'] != retry_times:
+ def _test_retry(self, req, exception, max_retry_times):
+
+ while max_retry_times > 0:
req = self.mw.process_exception(req, exception, self.spider)
assert isinstance(req, Request)
-
- self.assertEqual(req.meta['retry_times'], retry_times)
+ if req.meta['retry_times'] == max_retry_times:
+ break
# discard it
req = self.mw.process_exception(req, exception, self.spider)