Scrapy, a fast high-level web crawling & scraping framework for Python.
Implement retry request functions and mixin
Adrián Chavesa752fa072e1a
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 51fe59254..023ab7d60 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -31,6 +31,105 @@ from scrapy.utils.python import global_object_namelogger = logging.getLogger(__name__)+def get_retry_request(+ request,+ *,+ reason,+ spider,+ max_retry_times=None,+ priority_adjust=None,+):+ settings = spider.crawler.settings+ stats = spider.crawler.stats+ retry_times = request.meta.get('retry_times', 0) + 1+ request_max_retry_times = request.meta.get(+ 'max_retry_times',+ max_retry_times,+ )+ if request_max_retry_times is None:+ request_max_retry_times = settings.getint('RETRY_TIMES')+ if retry_times <= request_max_retry_times:+ logger.debug(+ "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",+ {'request': request, 'retry_times': retry_times, 'reason': reason},+ extra={'spider': spider}+ )+ new_request = request.copy()+ new_request.meta['retry_times'] = retry_times+ new_request.dont_filter = True+ if priority_adjust is None:+ priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')+ new_request.priority = request.priority + priority_adjust++ if isinstance(reason, Exception):+ reason = global_object_name(reason.__class__)++ stats.inc_value('retry/count')+ stats.inc_value(f'retry/reason_count/{reason}')+ return new_request+ else:+ stats.inc_value('retry/max_reached')+ logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",+ {'request': request, 'retry_times': retry_times, 'reason': reason},+ extra={'spider': spider})+ return None+++def retry_request(+ request,+ *,+ reason,+ spider,+ max_retry_times=None,+ priority_adjust=None,+):+ new_request = get_retry_request(+ request,+ reason=reason,+ spider=spider,+ max_retry_times=max_retry_times,+ priority_adjust=priority_adjust,+ )+ if new_request:+ return [new_request]+ return []+++class RetrySpiderMixin:++ def get_retry_request(+ self,+ request,+ *,+ reason,+ max_retry_times=None,+ priority_adjust=None,+ ):+ return get_retry_request(+ request,+ reason=reason,+ spider=self,+ max_retry_times=max_retry_times,+ priority_adjust=priority_adjust,+ )++ def retry_request(+ self,+ request,+ *,+ reason,+ max_retry_times=None,+ priority_adjust=None,+ ):+ return retry_request(+ request,+ reason=reason,+ spider=self,+ max_retry_times=max_retry_times,+ priority_adjust=priority_adjust,+ )++class RetryMiddleware:# IOError is raised by the HttpCompression middleware when trying to@@ -67,31 +166,10 @@ class RetryMiddleware:return self._retry(request, exception, spider)def _retry(self, request, reason, spider):- retries = request.meta.get('retry_times', 0) + 1-- retry_times = self.max_retry_times-- if 'max_retry_times' in request.meta:- retry_times = request.meta['max_retry_times']-- stats = spider.crawler.stats- if retries <= retry_times:- logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",- {'request': request, 'retries': retries, 'reason': reason},- extra={'spider': spider})- retryreq = request.copy()- retryreq.meta['retry_times'] = retries- retryreq.dont_filter = True- retryreq.priority = request.priority + self.priority_adjust-- if isinstance(reason, Exception):- reason = global_object_name(reason.__class__)-- stats.inc_value('retry/count')- stats.inc_value(f'retry/reason_count/{reason}')- return retryreq- else:- stats.inc_value('retry/max_reached')- logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",- {'request': request, 'retries': retries, 'reason': reason},- extra={'spider': spider})+ return get_retry_request(+ request,+ reason=reason,+ spider=spider,+ max_retry_times=self.max_retry_times,+ priority_adjust=self.priority_adjust,+ )