Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve the backwards compatibility for RetryMiddleware.EXCEPTIONS_TO_RETRY.

Andrey Rakhmatullin922ff5738448
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 205bb48b1..af590be47 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -23,12 +23,13 @@ from scrapy.utils.response import response_status_message
retry_logger = getLogger(__name__)
+DEPRECATED_ATTRIBUTE = "EXCEPTIONS_TO_RETRY"
-class BackwardsCompatibilityMetaclass(type):
- @property
- def EXCEPTIONS_TO_RETRY(cls):
+
+def backwards_compatibility_getattr(self, name):
+ if name == DEPRECATED_ATTRIBUTE:
warnings.warn(
- "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
+ f"Attribute RetryMiddleware.{DEPRECATED_ATTRIBUTE} is deprecated. "
"Use the RETRY_EXCEPTIONS setting instead.",
ScrapyDeprecationWarning,
stacklevel=2,
@@ -37,6 +38,13 @@ class BackwardsCompatibilityMetaclass(type):
load_object(x) if isinstance(x, str) else x
for x in Settings().getlist("RETRY_EXCEPTIONS")
)
+ raise AttributeError(
+ f"{self.__class__.__name__!r} object has no attribute {name!r}"
+ )
+
+
+class BackwardsCompatibilityMetaclass(type):
+ __getattr__ = backwards_compatibility_getattr
def get_retry_request(
@@ -137,15 +145,14 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
)
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
- if not hasattr(
- self, "EXCEPTIONS_TO_RETRY"
- ): # If EXCEPTIONS_TO_RETRY is not "overriden"
+ try:
+ self.exceptions_to_retry = self.__getattribute__(DEPRECATED_ATTRIBUTE)
+ except AttributeError:
+ # If EXCEPTIONS_TO_RETRY is not "overridden"
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
- else:
- self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
@classmethod
def from_crawler(cls, crawler):
@@ -175,3 +182,5 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)
+
+ __getattr__ = backwards_compatibility_getattr
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 97ae1e29a..661175840 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -122,7 +122,7 @@ class RetryTest(unittest.TestCase):
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
- def test_exception_to_retry_customMiddleware(self):
+ def test_exception_to_retry_custom_middleware(self):
exc = ValueError
with warnings.catch_warnings(record=True) as warns:
@@ -138,6 +138,21 @@ class RetryTest(unittest.TestCase):
assert isinstance(req, Request)
self.assertEqual(req.meta["retry_times"], 1)
+ def test_exception_to_retry_custom_middleware_self(self):
+ class MyRetryMiddleware(RetryMiddleware):
+ def process_exception(self, request, exception, spider):
+ if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
+ return self._retry(request, exception, spider)
+
+ exc = OSError
+ mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+ req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+ with warnings.catch_warnings(record=True) as warns:
+ req = mw2.process_exception(req, exc("foo"), self.spider)
+ assert isinstance(req, Request)
+ self.assertEqual(req.meta["retry_times"], 1)
+ self.assertEqual(len(warns), 1)
+
def _test_retry_exception(self, req, exception, mw=None):
if mw is None:
mw = self.mw