Scrapy, a fast high-level web crawling & scraping framework for Python.
Don't use CrawlerProcess in the commands that don't need it. (#6824)
Andrey Rakhmatulline0b9f2d8f6f0
diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex b11de2914..d90eb0bad 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -213,6 +213,8 @@ SpiderLoader API:param request: queried request:type request: :class:`~scrapy.Request` instance+.. autoclass:: DummySpiderLoader+.. _topics-api-signals:Signals APIdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 68c5079cf..65f2e5ebd 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exceptionBut you can choose to silence this exception and turn it into a simplewarning by setting ``SPIDER_LOADER_WARN_ONLY = True``.-.. note::- Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``- already (i.e. they will only issue a warning and will not fail)- since they do not actually need to load spider classes to work:- :command:`scrapy runspider <runspider>`,- :command:`scrapy settings <settings>`,- :command:`scrapy startproject <startproject>`,- :command:`scrapy version <version>`.-.. setting:: SPIDER_MIDDLEWARESSPIDER_MIDDLEWARESdiff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex b08fd3409..81e507a4e 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) ->opts, args = parser.parse_known_args(args=argv[1:])_run_print_help(parser, cmd.process_options, args, opts)- cmd.crawler_process = CrawlerProcess(settings)+ if cmd.requires_crawler_process:+ cmd.crawler_process = CrawlerProcess(settings)_run_print_help(parser, _run_command, cmd, args, opts)sys.exit(cmd.exitcode)diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 56199cc01..2818ead77 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -19,11 +19,13 @@ if TYPE_CHECKING:from collections.abc import Iterablefrom scrapy.crawler import Crawler, CrawlerProcess+ from scrapy.settings import Settingsclass ScrapyCommand:requires_project: bool = False- crawler_process: CrawlerProcess | None = None+ requires_crawler_process: bool = True+ crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline# default settings to be used for this command instead of global defaultsdefault_settings: dict[str, Any] = {}@@ -31,7 +33,7 @@ class ScrapyCommand:exitcode: int = 0def __init__(self) -> None:- self.settings: Any = None # set in scrapy.cmdline+ self.settings: Settings | None = None # set in scrapy.cmdlinedef set_crawler(self, crawler: Crawler) -> None:if hasattr(self, "_crawler"):@@ -68,6 +70,7 @@ class ScrapyCommand:"""Populate option parse with options available for this command"""+ assert self.settings is not Nonegroup = parser.add_argument_group(title="Global Options")group.add_argument("--logfile", metavar="FILE", help="log file. if omitted stderr will be used"@@ -100,6 +103,7 @@ class ScrapyCommand:group.add_argument("--pdb", action="store_true", help="enable pdb on failure")def process_options(self, args: list[str], opts: argparse.Namespace) -> None:+ assert self.settings is not Nonetry:self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")except ValueError:@@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand):except ValueError:raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)if opts.output or opts.overwrite_output:+ assert self.settings is not Nonefeeds = feed_process_params_from_cli(self.settings,opts.output,diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex 56dc1ea55..e9ada0fb6 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -69,6 +69,7 @@ class Command(ScrapyCommand):def run(self, args: list[str], opts: argparse.Namespace) -> None:# load contracts+ assert self.settings is not Nonecontracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))conman = ContractsManager(load_object(c) for c in contracts)runner = TextTestRunner(verbosity=2 if opts.verbose else 1)diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.pyindex d153a5271..f2d52673a 100644--- a/scrapy/commands/edit.py+++ b/scrapy/commands/edit.py@@ -4,10 +4,12 @@ import sysfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageError+from scrapy.spiderloader import get_spider_loaderclass Command(ScrapyCommand):requires_project = True+ requires_crawler_process = Falsedefault_settings = {"LOG_ENABLED": False}def syntax(self) -> str:@@ -30,10 +32,11 @@ class Command(ScrapyCommand):if len(args) != 1:raise UsageError+ assert self.settings is not Noneeditor = self.settings["EDITOR"]- assert self.crawler_process+ spider_loader = get_spider_loader(self.settings)try:- spidercls = self.crawler_process.spider_loader.load(args[0])+ spidercls = spider_loader.load(args[0])except KeyError:self._err(f"Spider not found: {args[0]}")returndiff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 6d4aec3d8..c4abfc4c9 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -11,6 +11,7 @@ from urllib.parse import urlparseimport scrapyfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageError+from scrapy.spiderloader import get_spider_loaderfrom scrapy.utils.template import render_templatefile, string_camelcaseif TYPE_CHECKING:@@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str:class Command(ScrapyCommand):requires_project = False+ requires_crawler_process = Falsedefault_settings = {"LOG_ENABLED": False}def syntax(self) -> str:@@ -92,6 +94,7 @@ class Command(ScrapyCommand):)def run(self, args: list[str], opts: argparse.Namespace) -> None:+ assert self.settings is not Noneif opts.list:self._list_templates()return@@ -127,6 +130,7 @@ class Command(ScrapyCommand):url: str,template_name: str,) -> dict[str, Any]:+ assert self.settings is not Nonecapitalized_module = "".join(s.capitalize() for s in module.split("_"))return {"project_name": self.settings.get("BOT_NAME"),@@ -147,6 +151,7 @@ class Command(ScrapyCommand):template_file: str | os.PathLike,) -> None:"""Generate the spider module, based on the given template"""+ assert self.settings is not Nonetvars = self._generate_template_variables(module, name, url, template_name)if self.settings.get("NEWSPIDER_MODULE"):spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])@@ -180,6 +185,7 @@ class Command(ScrapyCommand):print(f" {file.stem}")def _spider_exists(self, name: str) -> bool:+ assert self.settings is not Noneif not self.settings.get("NEWSPIDER_MODULE"):# if run as a standalone command and file with same filename already existspath = Path(name + ".py")@@ -188,12 +194,9 @@ class Command(ScrapyCommand):return Truereturn False- assert self.crawler_process is not None, (- "crawler_process must be set before calling run"- )-+ spider_loader = get_spider_loader(self.settings)try:- spidercls = self.crawler_process.spider_loader.load(name)+ spidercls = spider_loader.load(name)except KeyError:passelse:@@ -215,6 +218,7 @@ class Command(ScrapyCommand):@propertydef templates_dir(self) -> str:+ assert self.settings is not Nonereturn str(Path(self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),diff --git a/scrapy/commands/list.py b/scrapy/commands/list.pyindex 3b2f127c2..b4dc97f3d 100644--- a/scrapy/commands/list.py+++ b/scrapy/commands/list.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom scrapy.commands import ScrapyCommand+from scrapy.spiderloader import get_spider_loaderif TYPE_CHECKING:import argparse@@ -10,12 +11,14 @@ if TYPE_CHECKING:class Command(ScrapyCommand):requires_project = True+ requires_crawler_process = Falsedefault_settings = {"LOG_ENABLED": False}def short_desc(self) -> str:return "List available spiders"def run(self, args: list[str], opts: argparse.Namespace) -> None:- assert self.crawler_process- for s in sorted(self.crawler_process.spider_loader.list()):+ assert self.settings is not None+ spider_loader = get_spider_loader(self.settings)+ for s in sorted(spider_loader.list()):print(s)diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 357ca8b37..3e826456e 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -7,6 +7,7 @@ from typing import TYPE_CHECKINGfrom scrapy.commands import BaseRunSpiderCommandfrom scrapy.exceptions import UsageError+from scrapy.spiderloader import DummySpiderLoaderfrom scrapy.utils.spider import iter_spider_classesif TYPE_CHECKING:@@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:class Command(BaseRunSpiderCommand):requires_project = False- default_settings = {"SPIDER_LOADER_WARN_ONLY": True}+ default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}def syntax(self) -> str:return "[options] <spider_file>"diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex 59f86b9a7..e63031f2d 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettingsclass Command(ScrapyCommand):requires_project = False- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}+ requires_crawler_process = False+ default_settings = {"LOG_ENABLED": False}def syntax(self) -> str:return "[options]"@@ -46,8 +47,8 @@ class Command(ScrapyCommand):)def run(self, args: list[str], opts: argparse.Namespace) -> None:- assert self.crawler_process- settings = self.crawler_process.settings+ assert self.settings is not None+ settings = self.settingsif opts.get:s = settings.get(opts.get)if isinstance(s, BaseSettings):diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 1adc1530f..323979193 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None:class Command(ScrapyCommand):requires_project = False- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}+ requires_crawler_process = False+ default_settings = {"LOG_ENABLED": False}def syntax(self) -> str:return "<project_name> [project_dir]"@@ -132,6 +133,7 @@ class Command(ScrapyCommand):@propertydef templates_dir(self) -> str:+ assert self.settings is not Nonereturn str(Path(self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),diff --git a/scrapy/commands/version.py b/scrapy/commands/version.pyindex 713a78ad9..30b0e9fd7 100644--- a/scrapy/commands/version.py+++ b/scrapy/commands/version.py@@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versionsclass Command(ScrapyCommand):- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}+ requires_crawler_process = False+ default_settings = {"LOG_ENABLED": False}def syntax(self) -> str:return "[-v]"diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex c22b8603b..8e3223a5c 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -5,22 +5,21 @@ import contextlibimport loggingimport pprintimport signal-from typing import TYPE_CHECKING, Any, TypeVar, cast+from typing import TYPE_CHECKING, Any, TypeVarfrom twisted.internet.defer import (Deferred,DeferredList,inlineCallbacks,)-from zope.interface.verify import verifyClassfrom scrapy import Spider, signalsfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEnginefrom scrapy.extension import ExtensionManager-from scrapy.interfaces import ISpiderLoader-from scrapy.settings import BaseSettings, Settings, overridden_settings+from scrapy.settings import Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager+from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loaderfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import deferred_from_coro, deferred_to_futurefrom scrapy.utils.log import (@@ -46,7 +45,6 @@ if TYPE_CHECKING:from collections.abc import Generator, Iterablefrom scrapy.logformatter import LogFormatter- from scrapy.spiderloader import SpiderLoaderProtocolfrom scrapy.statscollectors import StatsCollectorfrom scrapy.utils.request import RequestFingerprinterProtocol@@ -324,22 +322,12 @@ class Crawler:class CrawlerRunnerBase:- @staticmethod- def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:- """Get SpiderLoader instance from settings"""- cls_path = settings.get("SPIDER_LOADER_CLASS")- loader_cls = load_object(cls_path)- verifyClass(ISpiderLoader, loader_cls)- return cast(- "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())- )-def __init__(self, settings: dict[str, Any] | Settings | None = None):if isinstance(settings, dict) or settings is None:settings = Settings(settings)AddonManager.load_pre_crawler_settings(settings)self.settings: Settings = settings- self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)+ self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings)self._crawlers: set[Crawler] = set()self.bootstrap_failed = Falsediff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex f537e0593..8eac188c8 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -3,12 +3,13 @@ from __future__ import annotationsimport tracebackimport warningsfrom collections import defaultdict-from typing import TYPE_CHECKING, Protocol+from typing import TYPE_CHECKING, Protocol, castfrom zope.interface import implementer+from zope.interface.verify import verifyClassfrom scrapy.interfaces import ISpiderLoader-from scrapy.utils.misc import walk_modules+from scrapy.utils.misc import load_object, walk_modulesfrom scrapy.utils.spider import iter_spider_classesif TYPE_CHECKING:@@ -21,6 +22,14 @@ if TYPE_CHECKING:from scrapy.settings import BaseSettings+def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:+ """Get SpiderLoader instance from settings"""+ cls_path = settings.get("SPIDER_LOADER_CLASS")+ loader_cls = load_object(cls_path)+ verifyClass(ISpiderLoader, loader_cls)+ return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))++class SpiderLoaderProtocol(Protocol):@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:@@ -120,3 +129,21 @@ class SpiderLoader:Return a list with the names of all spiders available in the project."""return list(self._spiders.keys())+++@implementer(ISpiderLoader)+class DummySpiderLoader:+ """A dummy spider loader that does not load any spiders."""++ @classmethod+ def from_settings(cls, settings: BaseSettings) -> Self:+ return cls()++ def load(self, spider_name: str) -> type[Spider]:+ raise KeyError("DummySpiderLoader doesn't load any spiders")++ def list(self) -> list[str]:+ return []++ def find_by_request(self, request: Request) -> __builtins__.list[str]:+ return []diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex ce5963cd7..56cb21650 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -30,7 +30,6 @@ from scrapy.crawler import (from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settings-from scrapy.spiderloader import SpiderLoaderfrom scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_corofrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpider@@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface:pass-class CustomSpiderLoader(SpiderLoader):- pass--class TestCrawlerRunner(TestBaseCrawler):def test_spider_manager_verify_interface(self):settings = Settings(diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 476487a04..245507c0b 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -1,10 +1,8 @@import contextlibimport shutilimport sys-import tempfileimport warningsfrom pathlib import Path-from tempfile import mkdtempfrom unittest import mockimport pytest@@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunnerfrom scrapy.http import Requestfrom scrapy.interfaces import ISpiderLoaderfrom scrapy.settings import Settings-from scrapy.spiderloader import SpiderLoader+from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loadermodule_dir = Path(__file__).resolve().parent@@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path):shutil.copytree(source, target)+def spider_loader_env(tmp_path):+ orig_spiders_dir = module_dir / "test_spiders"+ spiders_dir = tmp_path / "test_spiders_xxx"+ _copytree(orig_spiders_dir, spiders_dir)+ sys.path.append(str(tmp_path))+ settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})++ yield settings, spiders_dir++ sys.modules.pop("test_spiders_xxx", None)+ sys.path.remove(str(tmp_path))+++def spider_loader(spider_loader_env):+ settings, _ = spider_loader_env+ return SpiderLoader.from_settings(settings)++class TestSpiderLoader:- def setup_method(self):- orig_spiders_dir = module_dir / "test_spiders"- self.tmpdir = Path(tempfile.mkdtemp())- self.spiders_dir = self.tmpdir / "test_spiders_xxx"- _copytree(orig_spiders_dir, self.spiders_dir)- sys.path.append(str(self.tmpdir))- settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})- self.spider_loader = SpiderLoader.from_settings(settings)-- def teardown_method(self):- del self.spider_loader- del sys.modules["test_spiders_xxx"]- sys.path.remove(str(self.tmpdir))-- def test_interface(self):- verifyObject(ISpiderLoader, self.spider_loader)-- def test_list(self):- assert set(self.spider_loader.list()) == {+ def test_interface(self, spider_loader):+ verifyObject(ISpiderLoader, spider_loader)++ def test_list(self, spider_loader):+ assert set(spider_loader.list()) == {"spider1","spider2","spider3","spider4",}- def test_load(self):- spider1 = self.spider_loader.load("spider1")+ def test_load(self, spider_loader):+ spider1 = spider_loader.load("spider1")assert spider1.__name__ == "Spider1"- def test_find_by_request(self):- assert self.spider_loader.find_by_request(- Request("http://scrapy1.org/test")- ) == ["spider1"]- assert self.spider_loader.find_by_request(- Request("http://scrapy2.org/test")- ) == ["spider2"]+ def test_find_by_request(self, spider_loader):+ assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [+ "spider1"+ ]+ assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [+ "spider2"+ ]assert set(- self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))+ spider_loader.find_by_request(Request("http://scrapy3.org/test"))) == {"spider1", "spider2"}- assert (- self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))- == []- )- assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []- assert self.spider_loader.find_by_request(+ assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == []+ assert spider_loader.find_by_request(Request("http://spider3.com")) == []+ assert spider_loader.find_by_request(Request("http://spider3.com/onlythis")) == ["spider3"]def test_load_spider_module(self):module = "tests.test_spiderloader.test_spiders.spider1"settings = Settings({"SPIDER_MODULES": [module]})- self.spider_loader = SpiderLoader.from_settings(settings)- assert len(self.spider_loader._spiders) == 1+ spider_loader = SpiderLoader.from_settings(settings)+ assert len(spider_loader._spiders) == 1def test_load_spider_module_multiple(self):prefix = "tests.test_spiderloader.test_spiders."module = ",".join(prefix + s for s in ("spider1", "spider2"))settings = Settings({"SPIDER_MODULES": module})- self.spider_loader = SpiderLoader.from_settings(settings)- assert len(self.spider_loader._spiders) == 2+ spider_loader = SpiderLoader.from_settings(settings)+ assert len(spider_loader._spiders) == 2def test_load_base_spider(self):module = "tests.test_spiderloader.test_spiders.spider0"settings = Settings({"SPIDER_MODULES": [module]})- self.spider_loader = SpiderLoader.from_settings(settings)- assert len(self.spider_loader._spiders) == 0+ spider_loader = SpiderLoader.from_settings(settings)+ assert len(spider_loader._spiders) == 0def test_load_spider_module_from_addons(self):module = "tests.test_spiderloader.spiders_from_addons.spider0"@@ -183,27 +184,14 @@ class TestSpiderLoader:class TestDuplicateSpiderNameLoader:- def setup_method(self):- orig_spiders_dir = module_dir / "test_spiders"- self.tmpdir = Path(mkdtemp())- self.spiders_dir = self.tmpdir / "test_spiders_xxx"- _copytree(orig_spiders_dir, self.spiders_dir)- sys.path.append(str(self.tmpdir))- self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})-- def teardown_method(self):- del sys.modules["test_spiders_xxx"]- sys.path.remove(str(self.tmpdir))-- def test_dupename_warning(self):+ def test_dupename_warning(self, spider_loader_env):+ settings, spiders_dir = spider_loader_env+# copy 1 spider module so as to have duplicate spider name- shutil.copyfile(- self.tmpdir / "test_spiders_xxx" / "spider3.py",- self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",- )+ shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")with warnings.catch_warnings(record=True) as w:- spider_loader = SpiderLoader.from_settings(self.settings)+ spider_loader = SpiderLoader.from_settings(settings)assert len(w) == 1msg = str(w[0].message)@@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader:spiders = set(spider_loader.list())assert spiders == {"spider1", "spider2", "spider3", "spider4"}- def test_multiple_dupename_warning(self):+ def test_multiple_dupename_warning(self, spider_loader_env):+ settings, spiders_dir = spider_loader_env# copy 2 spider modules so as to have duplicate spider name# This should issue 2 warning, 1 for each duplicate spider name- shutil.copyfile(- self.tmpdir / "test_spiders_xxx" / "spider1.py",- self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",- )- shutil.copyfile(- self.tmpdir / "test_spiders_xxx" / "spider2.py",- self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",- )+ shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")+ shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")with warnings.catch_warnings(record=True) as w:- spider_loader = SpiderLoader.from_settings(self.settings)+ spider_loader = SpiderLoader.from_settings(settings)assert len(w) == 1msg = str(w[0].message)@@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader:spiders = set(spider_loader.list())assert spiders == {"spider1", "spider2", "spider3", "spider4"}+++class CustomSpiderLoader(SpiderLoader):+ pass+++def test_custom_spider_loader():+ settings = Settings(+ {+ "SPIDER_LOADER_CLASS": CustomSpiderLoader,+ }+ )+ spider_loader = get_spider_loader(settings)+ assert isinstance(spider_loader, CustomSpiderLoader)+++def test_dummy_spider_loader(spider_loader_env):+ settings, _ = spider_loader_env+ spider_loader = DummySpiderLoader.from_settings(settings)+ assert not spider_loader.list()+ with pytest.raises(KeyError):+ spider_loader.load("spider1")