Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add feature to set RETRY_TIMES per request (#2642)

harshasrinivas810658bcc5b1
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 549d74f46..a5342995f 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,6 +63,9 @@ class RetryMiddleware(object):
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
+ if 'max_retry_times' in request.meta:
+ self.max_retry_times = request.meta['max_retry_times']
+
stats = spider.crawler.stats
if retries <= self.max_retry_times:
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",