Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Prevent max_retry_times override

harshasrinivas0d57b5cd4334
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index a5342995f..07e979628 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,11 +63,13 @@ class RetryMiddleware(object):
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
+ retry_times = self.max_retry_times
+
if 'max_retry_times' in request.meta:
- self.max_retry_times = request.meta['max_retry_times']
+ retry_times = request.meta['max_retry_times']
stats = spider.crawler.stats
- if retries <= self.max_retry_times:
+ if retries <= retry_times:
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})