Scrapy, a fast high-level web crawling & scraping framework for Python.
Add feature to set RETRY_TIMES per request (#2642)
harshasrinivas810658bcc5b1
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 549d74f46..a5342995f 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -63,6 +63,9 @@ class RetryMiddleware(object):def _retry(self, request, reason, spider):retries = request.meta.get('retry_times', 0) + 1+ if 'max_retry_times' in request.meta:+ self.max_retry_times = request.meta['max_retry_times']+stats = spider.crawler.statsif retries <= self.max_retry_times:logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",