Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow logger/stats customization in get_retry_request
Eugenio Lacuesta9e62355271fa
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 0f24e5d28..8955c7e4f 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -29,7 +29,8 @@ from scrapy.utils.response import response_status_messagefrom scrapy.core.downloader.handlers.http11 import TunnelErrorfrom scrapy.utils.python import global_object_name-logger = logging.getLogger(__name__)++retry_logger = logging.getLogger(__name__)def get_retry_request(@@ -39,6 +40,8 @@ def get_retry_request(reason='unspecified',max_retry_times=None,priority_adjust=None,+ logger=retry_logger,+ stats_base_key='retry',):"""Returns a new :class:`~scrapy.Request` object to retry the specified@@ -76,6 +79,11 @@ def get_retry_request(*priority_adjust* is a number that determines how the priority of the newrequest changes in relation to *request*. If not specified, the number isread from the :setting:`RETRY_PRIORITY_ADJUST` setting.++ *logger* is the logging.Logger object to be used when logging messages++ *stats_base_key* is a string to be used as the base key for the+ retry-related job stats"""settings = spider.crawler.settingsstats = spider.crawler.stats@@ -102,11 +110,11 @@ def get_retry_request(if isinstance(reason, Exception):reason = global_object_name(reason.__class__)- stats.inc_value('retry/count')- stats.inc_value(f'retry/reason_count/{reason}')+ stats.inc_value(f'{stats_base_key}/count')+ stats.inc_value(f'{stats_base_key}/reason_count/{reason}')return new_requestelse:- stats.inc_value('retry/max_reached')+ stats.inc_value(f'{stats_base_key}/max_reached')logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): ""%(reason)s",diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 46e525f99..915bd3a3e 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -1,4 +1,7 @@+import loggingimport unittest++from testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.internet.error import (ConnectError,@@ -9,17 +12,12 @@ from twisted.internet.error import ()from twisted.web.client import ResponseFailed-from scrapy.downloadermiddlewares.retry import (- get_retry_request,- RetryMiddleware,-)+from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddlewarefrom scrapy.exceptions import IgnoreRequestfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-from testfixtures import LogCapture-class RetryTest(unittest.TestCase):def setUp(self):@@ -597,6 +595,40 @@ class GetRetryRequestTest(unittest.TestCase):))+ def test_custom_logger(self):+ logger = logging.getLogger("custom-logger")+ request = Request("https://example.com")+ spider = self.get_spider()+ expected_reason = "because"+ with LogCapture() as log:+ get_retry_request(+ request,+ spider=spider,+ reason=expected_reason,+ logger=logger,+ )+ log.check_present(+ (+ "custom-logger",+ "DEBUG",+ f"Retrying {request} (failed 1 times): {expected_reason}",+ )+ )++ def test_custom_stats_key(self):+ request = Request("https://example.com")+ spider = self.get_spider()+ expected_reason = "because"+ stats_key = "custom_retry"+ get_retry_request(+ request,+ spider=spider,+ reason=expected_reason,+ stats_base_key=stats_key,+ )+ for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):+ self.assertEqual(spider.crawler.stats.get_value(stat), 1)+if __name__ == "__main__":unittest.main()