Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a _load_objects() helper for object-or-path setting lists (#7857)
Adrian388d4fcf04e3
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex e1dcd90d5..f910d07c8 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -14,7 +14,7 @@ from typing import TYPE_CHECKINGfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.decorators import _warn_spider_arg-from scrapy.utils.misc import load_object+from scrapy.utils.misc import _load_objectsfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import response_status_message@@ -149,10 +149,7 @@ class RetryMiddleware:self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]- self.exceptions_to_retry = tuple(- load_object(x) if isinstance(x, str) else x- for x in settings.getlist("RETRY_EXCEPTIONS")- )+ self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS"))@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 20e7cb381..57b526be6 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:return obj+def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:+ """Resolve *objects* (objects or import paths) to a tuple of objects."""+ return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)++def walk_modules_iter(path: str) -> Iterable[ModuleType]:"""Loads a module and all its submodules from the given module path andreturns them. If *any* module throws an exception while importing, that