Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a _load_objects() helper for object-or-path setting lists (#7857)

Adrian388d4fcf04e3
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index e1dcd90d5..f910d07c8 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import _load_objects
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@@ -149,10 +149,7 @@ class RetryMiddleware:
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
- self.exceptions_to_retry = tuple(
- load_object(x) if isinstance(x, str) else x
- for x in settings.getlist("RETRY_EXCEPTIONS")
- )
+ self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS"))
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 20e7cb381..57b526be6 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:
return obj
+def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:
+ """Resolve *objects* (objects or import paths) to a tuple of objects."""
+ return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)
+
+
def walk_modules_iter(path: str) -> Iterable[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that