Scrapy, a fast high-level web crawling & scraping framework for Python.
deprecate `walk_modules` in favour of `walk_modules_iter` (#7388)
Albert Eduardovich N.ed31dcbb100e
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex 487980abd..057ddba4d 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -12,7 +12,7 @@ import scrapyfrom scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatterfrom scrapy.crawler import AsyncCrawlerProcess, CrawlerProcessfrom scrapy.exceptions import UsageError-from scrapy.utils.misc import walk_modules+from scrapy.utils.misc import walk_modules_iterfrom scrapy.utils.project import get_project_settings, inside_projectfrom scrapy.utils.python import garbage_collectfrom scrapy.utils.reactor import _asyncio_reactor_path@@ -40,7 +40,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]:# TODO: add `name` attribute to commands and merge this function with# scrapy.utils.spider.iter_spider_classes- for module in walk_modules(module_name):+ for module in walk_modules_iter(module_name):for obj in vars(module).values():if (inspect.isclass(obj)diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex 8eac188c8..c37b2e759 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -9,7 +9,7 @@ from zope.interface import implementerfrom zope.interface.verify import verifyClassfrom scrapy.interfaces import ISpiderLoader-from scrapy.utils.misc import load_object, walk_modules+from scrapy.utils.misc import load_object, walk_modules_iterfrom scrapy.utils.spider import iter_spider_classesif TYPE_CHECKING:@@ -88,7 +88,7 @@ class SpiderLoader:def _load_all_spiders(self) -> None:for name in self.spider_modules:try:- for module in walk_modules(name):+ for module in walk_modules_iter(name):self._load_spiders(module)except (ImportError, SyntaxError):if self.warn_only:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 7994a2471..ee988a665 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -77,26 +77,49 @@ def load_object(path: str | Callable[..., Any]) -> Any:return obj-def walk_modules(path: str) -> list[ModuleType]:+def walk_modules_iter(path: str) -> Iterable[ModuleType]:"""Loads a module and all its submodules from the given module path andreturns them. If *any* module throws an exception while importing, thatexception is thrown back.- For example: walk_modules('scrapy.utils')+ For example:+ >>> list(walk_modules_iter('scrapy.utils'))+ [<module 'scrapy.utils' from '...'>, ...]+ >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed+ >>> list(gen)+ Traceback (most recent call last):+ ...+ ModuleNotFoundError: No module named 'scrapy.utils.nonexistent'"""- mods: list[ModuleType] = []mod = import_module(path)- mods.append(mod)+ yield modif hasattr(mod, "__path__"):for _, subpath, ispkg in iter_modules(mod.__path__):fullpath = path + "." + subpathif ispkg:- mods += walk_modules(fullpath)+ yield from walk_modules_iter(fullpath)else:- submod = import_module(fullpath)- mods.append(submod)- return mods+ yield import_module(fullpath)+++def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover+ """+ Loads a module and all its submodules from the given module path and+ returns them. If *any* module throws an exception while importing, that+ exception is thrown back.+ """+ warnings.warn(+ (+ "The scrapy.utils.misc.walk_modules function is deprecated and will be "+ "removed in a future version of Scrapy. "+ "Use scrapy.utils.misc.walk_modules_iter instead."+ ),+ ScrapyDeprecationWarning,+ stacklevel=2,+ )++ return list(walk_modules_iter(path))def md5sum(file: IO[bytes]) -> str:diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex 610fc0ffe..a995e38e6 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -5,6 +5,7 @@ from unittest import mockimport pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.item import Field, Itemfrom scrapy.utils.misc import (arg_to_iter,@@ -13,6 +14,7 @@ from scrapy.utils.misc import (rel_has_nofollow,set_environ,walk_modules,+ walk_modules_iter,)@@ -38,7 +40,7 @@ class TestUtilsMisc:load_object({})def test_walk_modules(self):- mods = walk_modules("tests.test_utils_misc.test_walk_modules")+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")expected = ["tests.test_utils_misc.test_walk_modules","tests.test_utils_misc.test_walk_modules.mod",@@ -47,27 +49,38 @@ class TestUtilsMisc:]assert {m.__name__ for m in mods} == set(expected)- mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod")+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")expected = ["tests.test_utils_misc.test_walk_modules.mod","tests.test_utils_misc.test_walk_modules.mod.mod0",]assert {m.__name__ for m in mods} == set(expected)- mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1")+ mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")expected = ["tests.test_utils_misc.test_walk_modules.mod1",]assert {m.__name__ for m in mods} == set(expected)with pytest.raises(ImportError):+ for _ in walk_modules_iter("nomodule999"):+ pass+ with (+ pytest.raises(ImportError),+ pytest.warns(+ ScrapyDeprecationWarning,+ match="The scrapy.utils.misc.walk_modules function is deprecated and will be "+ "removed in a future version of Scrapy. "+ "Use scrapy.utils.misc.walk_modules_iter instead.",+ ),+ ):walk_modules("nomodule999")def test_walk_modules_egg(self):egg = str(Path(__file__).parent / "test.egg")sys.path.append(egg)try:- mods = walk_modules("testegg")+ mods = walk_modules_iter("testegg")expected = ["testegg.spiders","testegg.spiders.a",