Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow configuring the log level of the retry give-up message (#7567)
Adnan Awand2290c35c23b
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex a51e431d2..91ee38b85 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1053,6 +1053,21 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaughtexception propagation, see:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.+.. setting:: RETRY_GIVE_UP_LOG_LEVEL++RETRY_GIVE_UP_LOG_LEVEL+^^^^^^^^^^^^^^^^^^^^^^^++Default: ``"ERROR"``++:ref:`Logging level <levels>` used for the message logged when a request+exceeds its retries.++Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING``+or ``30``).++See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`.+.. setting:: RETRY_PRIORITY_ADJUSTRETRY_PRIORITY_ADJUSTdiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex a4f031803..8fd3de621 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -714,6 +714,7 @@ Those are:* :reqmeta:`download_timeout`* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)* ``ftp_user`` (See :setting:`FTP_USER` for more info)+* :reqmeta:`give_up_log_level`* :reqmeta:`handle_httpstatus_all`* :reqmeta:`handle_httpstatus_list`* :reqmeta:`is_start_request`@@ -790,12 +791,20 @@ download_fail_on_datalossWhether or not to fail on broken responses. See::setting:`DOWNLOAD_FAIL_ON_DATALOSS`.+.. reqmeta:: give_up_log_level++give_up_log_level+-----------------++:ref:`Logging level <levels>` used for the message logged when a request+exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.+.. reqmeta:: max_retry_timesmax_retry_times----------------The meta key is used set retry times per request. When initialized, the+The meta key is used set retry times per request. When set, the:reqmeta:`max_retry_times` meta key takes higher precedence over the:setting:`RETRY_TIMES` setting.diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex d38b4b9db..5f125cae4 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -12,7 +12,7 @@ once the spider has finished crawling all regular (non-failed) pages.from __future__ import annotations-from logging import Logger, getLogger+from logging import Logger, getLevelName, getLoggerfrom typing import TYPE_CHECKINGfrom scrapy.exceptions import NotConfigured@@ -43,6 +43,7 @@ def get_retry_request(max_retry_times: int | None = None,priority_adjust: int | None = None,logger: Logger = retry_logger,+ give_up_log_level: int | str | None = None,stats_base_key: str = "retry",) -> Request | None:"""@@ -51,14 +52,16 @@ def get_retry_request(exhausted.For example, in a :class:`~scrapy.Spider` callback, you could use it as- follows::+ follows:++ .. code-block:: pythondef parse(self, response):if not response.text:new_request_or_none = get_retry_request(response.request,spider=self,- reason='empty',+ reason="empty",)return new_request_or_none@@ -82,6 +85,10 @@ def get_retry_request(*logger* is the logging.Logger object to be used when logging messages+ *give_up_log_level* is the :ref:`logging level <levels>` used for the+ message logged when a request exceeds its retries. See+ :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.+*stats_base_key* is a string to be used as the base key for theretry-related job stats"""@@ -114,8 +121,16 @@ def get_retry_request(stats.inc_value(f"{stats_base_key}/count")stats.inc_value(f"{stats_base_key}/reason_count/{reason}")return new_request+ if give_up_log_level is None:+ give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]+ if isinstance(give_up_log_level, str):+ level = getLevelName(give_up_log_level)+ if not isinstance(level, int):+ raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}")+ give_up_log_level = levelstats.inc_value(f"{stats_base_key}/max_reached")- logger.error(+ logger.log(+ give_up_log_level,"Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",{"request": request, "retry_times": retry_times, "reason": reason},extra={"spider": spider},@@ -132,6 +147,7 @@ class RetryMiddleware:self.max_retry_times = settings.getint("RETRY_TIMES")self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")+ self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]self.exceptions_to_retry = tuple(load_object(x) if isinstance(x, str) else xfor x in settings.getlist("RETRY_EXCEPTIONS")@@ -175,6 +191,9 @@ class RetryMiddleware:) -> Request | None:max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)+ give_up_log_level = request.meta.get(+ "give_up_log_level", self.give_up_log_level+ )assert self.crawler.spiderreturn get_retry_request(request,@@ -182,4 +201,5 @@ class RetryMiddleware:spider=self.crawler.spider,max_retry_times=max_retry_times,priority_adjust=priority_adjust,+ give_up_log_level=give_up_log_level,)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex de03c0107..909a5fd5c 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -156,6 +156,7 @@ __all__ = ["REQUEST_FINGERPRINTER_CLASS","RETRY_ENABLED","RETRY_EXCEPTIONS",+ "RETRY_GIVE_UP_LOG_LEVEL","RETRY_HTTP_CODES","RETRY_PRIORITY_ADJUST","RETRY_TIMES",@@ -469,6 +470,7 @@ RETRY_EXCEPTIONS = [OSError,"scrapy.core.downloader.handlers.http11.TunnelError",]+RETRY_GIVE_UP_LOG_LEVEL = "ERROR"RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]RETRY_PRIORITY_ADJUST = -1RETRY_TIMES = 2 # initial response + 2 retries = 3 requestsdiff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 50946899a..56d21a4d2 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -84,6 +84,39 @@ class TestRetry:)assert self.crawler.stats.get_value("retry/count") == 2+ def test_give_up_log_level_setting(self):+ crawler = get_crawler(+ DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}+ )+ crawler.spider = crawler._create_spider()+ mw = RetryMiddleware.from_crawler(crawler)+ mw.max_retry_times = 0+ req = Request("http://example.com/503")+ rsp = Response("http://example.com/503", body=b"", status=503)+ with LogCapture() as log:+ assert mw.process_response(req, rsp) is rsp+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "WARNING",+ f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",+ )+ )++ def test_give_up_log_level_meta(self):+ self.mw.max_retry_times = 0+ req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"})+ rsp = Response("http://example.com/503", body=b"", status=503)+ with LogCapture() as log:+ assert self.mw.process_response(req, rsp) is rsp+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "WARNING",+ f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",+ )+ )+def test_twistederrors(self):exceptions = [ConnectError,@@ -612,6 +645,87 @@ class TestGetRetryRequest:))+ def test_give_up_log_level_default(self):+ request = Request("https://example.com")+ spider = self.get_spider()+ with LogCapture() as log:+ get_retry_request(+ request,+ spider=spider,+ max_retry_times=0,+ )+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "ERROR",+ f"Gave up retrying {request} (failed 1 times): unspecified",+ )+ )++ def test_give_up_log_level_argument_name(self):+ request = Request("https://example.com")+ spider = self.get_spider()+ with LogCapture() as log:+ get_retry_request(+ request,+ spider=spider,+ max_retry_times=0,+ give_up_log_level="WARNING",+ )+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "WARNING",+ f"Gave up retrying {request} (failed 1 times): unspecified",+ )+ )++ def test_give_up_log_level_argument_number(self):+ request = Request("https://example.com")+ spider = self.get_spider()+ with LogCapture() as log:+ get_retry_request(+ request,+ spider=spider,+ max_retry_times=0,+ give_up_log_level=logging.WARNING,+ )+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "WARNING",+ f"Gave up retrying {request} (failed 1 times): unspecified",+ )+ )++ def test_give_up_log_level_setting(self):+ request = Request("https://example.com")+ spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"})+ with LogCapture() as log:+ get_retry_request(+ request,+ spider=spider,+ max_retry_times=0,+ )+ log.check_present(+ (+ "scrapy.downloadermiddlewares.retry",+ "WARNING",+ f"Gave up retrying {request} (failed 1 times): unspecified",+ )+ )++ def test_give_up_log_level_invalid(self):+ request = Request("https://example.com")+ spider = self.get_spider()+ with pytest.raises(ValueError, match="Invalid give-up log level"):+ get_retry_request(+ request,+ spider=spider,+ max_retry_times=0,+ give_up_log_level="NOT_A_LEVEL",+ )+def test_custom_stats_key(self):request = Request("https://example.com")spider = self.get_spider()