Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve the backwards compatibility for RetryMiddleware.EXCEPTIONS_TO_RETRY.
Andrey Rakhmatullin922ff5738448
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 205bb48b1..af590be47 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -23,12 +23,13 @@ from scrapy.utils.response import response_status_messageretry_logger = getLogger(__name__)+DEPRECATED_ATTRIBUTE = "EXCEPTIONS_TO_RETRY"-class BackwardsCompatibilityMetaclass(type):- @property- def EXCEPTIONS_TO_RETRY(cls):++def backwards_compatibility_getattr(self, name):+ if name == DEPRECATED_ATTRIBUTE:warnings.warn(- "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "+ f"Attribute RetryMiddleware.{DEPRECATED_ATTRIBUTE} is deprecated. ""Use the RETRY_EXCEPTIONS setting instead.",ScrapyDeprecationWarning,stacklevel=2,@@ -37,6 +38,13 @@ class BackwardsCompatibilityMetaclass(type):load_object(x) if isinstance(x, str) else xfor x in Settings().getlist("RETRY_EXCEPTIONS"))+ raise AttributeError(+ f"{self.__class__.__name__!r} object has no attribute {name!r}"+ )+++class BackwardsCompatibilityMetaclass(type):+ __getattr__ = backwards_compatibility_getattrdef get_retry_request(@@ -137,15 +145,14 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):)self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")- if not hasattr(- self, "EXCEPTIONS_TO_RETRY"- ): # If EXCEPTIONS_TO_RETRY is not "overriden"+ try:+ self.exceptions_to_retry = self.__getattribute__(DEPRECATED_ATTRIBUTE)+ except AttributeError:+ # If EXCEPTIONS_TO_RETRY is not "overridden"self.exceptions_to_retry = tuple(load_object(x) if isinstance(x, str) else xfor x in settings.getlist("RETRY_EXCEPTIONS"))- else:- self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY@classmethoddef from_crawler(cls, crawler):@@ -175,3 +182,5 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):max_retry_times=max_retry_times,priority_adjust=priority_adjust,)++ __getattr__ = backwards_compatibility_getattrdiff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 97ae1e29a..661175840 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -122,7 +122,7 @@ class RetryTest(unittest.TestCase):req = Request(f"http://www.scrapytest.org/{exc.__name__}")self._test_retry_exception(req, exc("foo"), mw)- def test_exception_to_retry_customMiddleware(self):+ def test_exception_to_retry_custom_middleware(self):exc = ValueErrorwith warnings.catch_warnings(record=True) as warns:@@ -138,6 +138,21 @@ class RetryTest(unittest.TestCase):assert isinstance(req, Request)self.assertEqual(req.meta["retry_times"], 1)+ def test_exception_to_retry_custom_middleware_self(self):+ class MyRetryMiddleware(RetryMiddleware):+ def process_exception(self, request, exception, spider):+ if isinstance(exception, self.EXCEPTIONS_TO_RETRY):+ return self._retry(request, exception, spider)++ exc = OSError+ mw2 = MyRetryMiddleware.from_crawler(self.crawler)+ req = Request(f"http://www.scrapytest.org/{exc.__name__}")+ with warnings.catch_warnings(record=True) as warns:+ req = mw2.process_exception(req, exc("foo"), self.spider)+ assert isinstance(req, Request)+ self.assertEqual(req.meta["retry_times"], 1)+ self.assertEqual(len(warns), 1)+def _test_retry_exception(self, req, exception, mw=None):if mw is None:mw = self.mw