Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

deprecate `walk_modules` in favour of `walk_modules_iter` (#7388)

Albert Eduardovich N.ed31dcbb100e
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 487980abd..057ddba4d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -12,7 +12,7 @@ import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess
from scrapy.exceptions import UsageError
-from scrapy.utils.misc import walk_modules
+from scrapy.utils.misc import walk_modules_iter
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
from scrapy.utils.reactor import _asyncio_reactor_path
@@ -40,7 +40,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]:
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
- for module in walk_modules(module_name):
+ for module in walk_modules_iter(module_name):
for obj in vars(module).values():
if (
inspect.isclass(obj)
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 8eac188c8..c37b2e759 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -9,7 +9,7 @@ from zope.interface import implementer
from zope.interface.verify import verifyClass
from scrapy.interfaces import ISpiderLoader
-from scrapy.utils.misc import load_object, walk_modules
+from scrapy.utils.misc import load_object, walk_modules_iter
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
@@ -88,7 +88,7 @@ class SpiderLoader:
def _load_all_spiders(self) -> None:
for name in self.spider_modules:
try:
- for module in walk_modules(name):
+ for module in walk_modules_iter(name):
self._load_spiders(module)
except (ImportError, SyntaxError):
if self.warn_only:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 7994a2471..ee988a665 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -77,26 +77,49 @@ def load_object(path: str | Callable[..., Any]) -> Any:
return obj
-def walk_modules(path: str) -> list[ModuleType]:
+def walk_modules_iter(path: str) -> Iterable[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that
exception is thrown back.
- For example: walk_modules('scrapy.utils')
+ For example:
+ >>> list(walk_modules_iter('scrapy.utils'))
+ [<module 'scrapy.utils' from '...'>, ...]
+ >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed
+ >>> list(gen)
+ Traceback (most recent call last):
+ ...
+ ModuleNotFoundError: No module named 'scrapy.utils.nonexistent'
"""
- mods: list[ModuleType] = []
mod = import_module(path)
- mods.append(mod)
+ yield mod
if hasattr(mod, "__path__"):
for _, subpath, ispkg in iter_modules(mod.__path__):
fullpath = path + "." + subpath
if ispkg:
- mods += walk_modules(fullpath)
+ yield from walk_modules_iter(fullpath)
else:
- submod = import_module(fullpath)
- mods.append(submod)
- return mods
+ yield import_module(fullpath)
+
+
+def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover
+ """
+ Loads a module and all its submodules from the given module path and
+ returns them. If *any* module throws an exception while importing, that
+ exception is thrown back.
+ """
+ warnings.warn(
+ (
+ "The scrapy.utils.misc.walk_modules function is deprecated and will be "
+ "removed in a future version of Scrapy. "
+ "Use scrapy.utils.misc.walk_modules_iter instead."
+ ),
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
+ return list(walk_modules_iter(path))
def md5sum(file: IO[bytes]) -> str:
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 610fc0ffe..a995e38e6 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -5,6 +5,7 @@ from unittest import mock
import pytest
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
@@ -13,6 +14,7 @@ from scrapy.utils.misc import (
rel_has_nofollow,
set_environ,
walk_modules,
+ walk_modules_iter,
)
@@ -38,7 +40,7 @@ class TestUtilsMisc:
load_object({})
def test_walk_modules(self):
- mods = walk_modules("tests.test_utils_misc.test_walk_modules")
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
expected = [
"tests.test_utils_misc.test_walk_modules",
"tests.test_utils_misc.test_walk_modules.mod",
@@ -47,27 +49,38 @@ class TestUtilsMisc:
]
assert {m.__name__ for m in mods} == set(expected)
- mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod")
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
expected = [
"tests.test_utils_misc.test_walk_modules.mod",
"tests.test_utils_misc.test_walk_modules.mod.mod0",
]
assert {m.__name__ for m in mods} == set(expected)
- mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1")
+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
expected = [
"tests.test_utils_misc.test_walk_modules.mod1",
]
assert {m.__name__ for m in mods} == set(expected)
with pytest.raises(ImportError):
+ for _ in walk_modules_iter("nomodule999"):
+ pass
+ with (
+ pytest.raises(ImportError),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
+ "removed in a future version of Scrapy. "
+ "Use scrapy.utils.misc.walk_modules_iter instead.",
+ ),
+ ):
walk_modules("nomodule999")
def test_walk_modules_egg(self):
egg = str(Path(__file__).parent / "test.egg")
sys.path.append(egg)
try:
- mods = walk_modules("testegg")
+ mods = walk_modules_iter("testegg")
expected = [
"testegg.spiders",
"testegg.spiders.a",