Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't use CrawlerProcess in the commands that don't need it. (#6824)

Andrey Rakhmatulline0b9f2d8f6f0
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index b11de2914..d90eb0bad 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -213,6 +213,8 @@ SpiderLoader API
:param request: queried request
:type request: :class:`~scrapy.Request` instance
+.. autoclass:: DummySpiderLoader
+
.. _topics-api-signals:
Signals API
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 68c5079cf..65f2e5ebd 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
-.. note::
- Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
- already (i.e. they will only issue a warning and will not fail)
- since they do not actually need to load spider classes to work:
- :command:`scrapy runspider <runspider>`,
- :command:`scrapy settings <settings>`,
- :command:`scrapy startproject <startproject>`,
- :command:`scrapy version <version>`.
-
.. setting:: SPIDER_MIDDLEWARES
SPIDER_MIDDLEWARES
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b08fd3409..81e507a4e 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) ->
opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
- cmd.crawler_process = CrawlerProcess(settings)
+ if cmd.requires_crawler_process:
+ cmd.crawler_process = CrawlerProcess(settings)
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 56199cc01..2818ead77 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -19,11 +19,13 @@ if TYPE_CHECKING:
from collections.abc import Iterable
from scrapy.crawler import Crawler, CrawlerProcess
+ from scrapy.settings import Settings
class ScrapyCommand:
requires_project: bool = False
- crawler_process: CrawlerProcess | None = None
+ requires_crawler_process: bool = True
+ crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline
# default settings to be used for this command instead of global defaults
default_settings: dict[str, Any] = {}
@@ -31,7 +33,7 @@ class ScrapyCommand:
exitcode: int = 0
def __init__(self) -> None:
- self.settings: Any = None # set in scrapy.cmdline
+ self.settings: Settings | None = None # set in scrapy.cmdline
def set_crawler(self, crawler: Crawler) -> None:
if hasattr(self, "_crawler"):
@@ -68,6 +70,7 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
+ assert self.settings is not None
group = parser.add_argument_group(title="Global Options")
group.add_argument(
"--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
@@ -100,6 +103,7 @@ class ScrapyCommand:
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
+ assert self.settings is not None
try:
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
@@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output or opts.overwrite_output:
+ assert self.settings is not None
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 56dc1ea55..e9ada0fb6 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -69,6 +69,7 @@ class Command(ScrapyCommand):
def run(self, args: list[str], opts: argparse.Namespace) -> None:
# load contracts
+ assert self.settings is not None
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index d153a5271..f2d52673a 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -4,10 +4,12 @@ import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
+from scrapy.spiderloader import get_spider_loader
class Command(ScrapyCommand):
requires_project = True
+ requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
@@ -30,10 +32,11 @@ class Command(ScrapyCommand):
if len(args) != 1:
raise UsageError
+ assert self.settings is not None
editor = self.settings["EDITOR"]
- assert self.crawler_process
+ spider_loader = get_spider_loader(self.settings)
try:
- spidercls = self.crawler_process.spider_loader.load(args[0])
+ spidercls = spider_loader.load(args[0])
except KeyError:
self._err(f"Spider not found: {args[0]}")
return
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 6d4aec3d8..c4abfc4c9 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -11,6 +11,7 @@ from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
+from scrapy.spiderloader import get_spider_loader
from scrapy.utils.template import render_templatefile, string_camelcase
if TYPE_CHECKING:
@@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str:
class Command(ScrapyCommand):
requires_project = False
+ requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
@@ -92,6 +94,7 @@ class Command(ScrapyCommand):
)
def run(self, args: list[str], opts: argparse.Namespace) -> None:
+ assert self.settings is not None
if opts.list:
self._list_templates()
return
@@ -127,6 +130,7 @@ class Command(ScrapyCommand):
url: str,
template_name: str,
) -> dict[str, Any]:
+ assert self.settings is not None
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
return {
"project_name": self.settings.get("BOT_NAME"),
@@ -147,6 +151,7 @@ class Command(ScrapyCommand):
template_file: str | os.PathLike,
) -> None:
"""Generate the spider module, based on the given template"""
+ assert self.settings is not None
tvars = self._generate_template_variables(module, name, url, template_name)
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
@@ -180,6 +185,7 @@ class Command(ScrapyCommand):
print(f" {file.stem}")
def _spider_exists(self, name: str) -> bool:
+ assert self.settings is not None
if not self.settings.get("NEWSPIDER_MODULE"):
# if run as a standalone command and file with same filename already exists
path = Path(name + ".py")
@@ -188,12 +194,9 @@ class Command(ScrapyCommand):
return True
return False
- assert self.crawler_process is not None, (
- "crawler_process must be set before calling run"
- )
-
+ spider_loader = get_spider_loader(self.settings)
try:
- spidercls = self.crawler_process.spider_loader.load(name)
+ spidercls = spider_loader.load(name)
except KeyError:
pass
else:
@@ -215,6 +218,7 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
+ assert self.settings is not None
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 3b2f127c2..b4dc97f3d 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.commands import ScrapyCommand
+from scrapy.spiderloader import get_spider_loader
if TYPE_CHECKING:
import argparse
@@ -10,12 +11,14 @@ if TYPE_CHECKING:
class Command(ScrapyCommand):
requires_project = True
+ requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def short_desc(self) -> str:
return "List available spiders"
def run(self, args: list[str], opts: argparse.Namespace) -> None:
- assert self.crawler_process
- for s in sorted(self.crawler_process.spider_loader.list()):
+ assert self.settings is not None
+ spider_loader = get_spider_loader(self.settings)
+ for s in sorted(spider_loader.list()):
print(s)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 357ca8b37..3e826456e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
+from scrapy.spiderloader import DummySpiderLoader
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
@@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
class Command(BaseRunSpiderCommand):
requires_project = False
- default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
+ default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}
def syntax(self) -> str:
return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 59f86b9a7..e63031f2d 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+ requires_crawler_process = False
+ default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[options]"
@@ -46,8 +47,8 @@ class Command(ScrapyCommand):
)
def run(self, args: list[str], opts: argparse.Namespace) -> None:
- assert self.crawler_process
- settings = self.crawler_process.settings
+ assert self.settings is not None
+ settings = self.settings
if opts.get:
s = settings.get(opts.get)
if isinstance(s, BaseSettings):
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 1adc1530f..323979193 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None:
class Command(ScrapyCommand):
requires_project = False
- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+ requires_crawler_process = False
+ default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "<project_name> [project_dir]"
@@ -132,6 +133,7 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
+ assert self.settings is not None
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 713a78ad9..30b0e9fd7 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions
class Command(ScrapyCommand):
- default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+ requires_crawler_process = False
+ default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[-v]"
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index c22b8603b..8e3223a5c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -5,22 +5,21 @@ import contextlib
import logging
import pprint
import signal
-from typing import TYPE_CHECKING, Any, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar
from twisted.internet.defer import (
Deferred,
DeferredList,
inlineCallbacks,
)
-from zope.interface.verify import verifyClass
from scrapy import Spider, signals
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.extension import ExtensionManager
-from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import BaseSettings, Settings, overridden_settings
+from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
+from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import deferred_from_coro, deferred_to_future
from scrapy.utils.log import (
@@ -46,7 +45,6 @@ if TYPE_CHECKING:
from collections.abc import Generator, Iterable
from scrapy.logformatter import LogFormatter
- from scrapy.spiderloader import SpiderLoaderProtocol
from scrapy.statscollectors import StatsCollector
from scrapy.utils.request import RequestFingerprinterProtocol
@@ -324,22 +322,12 @@ class Crawler:
class CrawlerRunnerBase:
- @staticmethod
- def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
- """Get SpiderLoader instance from settings"""
- cls_path = settings.get("SPIDER_LOADER_CLASS")
- loader_cls = load_object(cls_path)
- verifyClass(ISpiderLoader, loader_cls)
- return cast(
- "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())
- )
-
def __init__(self, settings: dict[str, Any] | Settings | None = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
AddonManager.load_pre_crawler_settings(settings)
self.settings: Settings = settings
- self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)
+ self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings)
self._crawlers: set[Crawler] = set()
self.bootstrap_failed = False
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index f537e0593..8eac188c8 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -3,12 +3,13 @@ from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
-from typing import TYPE_CHECKING, Protocol
+from typing import TYPE_CHECKING, Protocol, cast
from zope.interface import implementer
+from zope.interface.verify import verifyClass
from scrapy.interfaces import ISpiderLoader
-from scrapy.utils.misc import walk_modules
+from scrapy.utils.misc import load_object, walk_modules
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
@@ -21,6 +22,14 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
+def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
+ """Get SpiderLoader instance from settings"""
+ cls_path = settings.get("SPIDER_LOADER_CLASS")
+ loader_cls = load_object(cls_path)
+ verifyClass(ISpiderLoader, loader_cls)
+ return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
+
+
class SpiderLoaderProtocol(Protocol):
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
@@ -120,3 +129,21 @@ class SpiderLoader:
Return a list with the names of all spiders available in the project.
"""
return list(self._spiders.keys())
+
+
+@implementer(ISpiderLoader)
+class DummySpiderLoader:
+ """A dummy spider loader that does not load any spiders."""
+
+ @classmethod
+ def from_settings(cls, settings: BaseSettings) -> Self:
+ return cls()
+
+ def load(self, spider_name: str) -> type[Spider]:
+ raise KeyError("DummySpiderLoader doesn't load any spiders")
+
+ def list(self) -> list[str]:
+ return []
+
+ def find_by_request(self, request: Request) -> __builtins__.list[str]:
+ return []
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ce5963cd7..56cb21650 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -30,7 +30,6 @@ from scrapy.crawler import (
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
-from scrapy.spiderloader import SpiderLoader
from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
@@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface:
pass
-class CustomSpiderLoader(SpiderLoader):
- pass
-
-
class TestCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self):
settings = Settings(
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 476487a04..245507c0b 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,10 +1,8 @@
import contextlib
import shutil
import sys
-import tempfile
import warnings
from pathlib import Path
-from tempfile import mkdtemp
from unittest import mock
import pytest
@@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import Settings
-from scrapy.spiderloader import SpiderLoader
+from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
module_dir = Path(__file__).resolve().parent
@@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path):
shutil.copytree(source, target)
+def spider_loader_env(tmp_path):
+ orig_spiders_dir = module_dir / "test_spiders"
+ spiders_dir = tmp_path / "test_spiders_xxx"
+ _copytree(orig_spiders_dir, spiders_dir)
+ sys.path.append(str(tmp_path))
+ settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
+
+ yield settings, spiders_dir
+
+ sys.modules.pop("test_spiders_xxx", None)
+ sys.path.remove(str(tmp_path))
+
+
+def spider_loader(spider_loader_env):
+ settings, _ = spider_loader_env
+ return SpiderLoader.from_settings(settings)
+
+
class TestSpiderLoader:
- def setup_method(self):
- orig_spiders_dir = module_dir / "test_spiders"
- self.tmpdir = Path(tempfile.mkdtemp())
- self.spiders_dir = self.tmpdir / "test_spiders_xxx"
- _copytree(orig_spiders_dir, self.spiders_dir)
- sys.path.append(str(self.tmpdir))
- settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
- self.spider_loader = SpiderLoader.from_settings(settings)
-
- def teardown_method(self):
- del self.spider_loader
- del sys.modules["test_spiders_xxx"]
- sys.path.remove(str(self.tmpdir))
-
- def test_interface(self):
- verifyObject(ISpiderLoader, self.spider_loader)
-
- def test_list(self):
- assert set(self.spider_loader.list()) == {
+ def test_interface(self, spider_loader):
+ verifyObject(ISpiderLoader, spider_loader)
+
+ def test_list(self, spider_loader):
+ assert set(spider_loader.list()) == {
"spider1",
"spider2",
"spider3",
"spider4",
}
- def test_load(self):
- spider1 = self.spider_loader.load("spider1")
+ def test_load(self, spider_loader):
+ spider1 = spider_loader.load("spider1")
assert spider1.__name__ == "Spider1"
- def test_find_by_request(self):
- assert self.spider_loader.find_by_request(
- Request("http://scrapy1.org/test")
- ) == ["spider1"]
- assert self.spider_loader.find_by_request(
- Request("http://scrapy2.org/test")
- ) == ["spider2"]
+ def test_find_by_request(self, spider_loader):
+ assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [
+ "spider1"
+ ]
+ assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [
+ "spider2"
+ ]
assert set(
- self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
+ spider_loader.find_by_request(Request("http://scrapy3.org/test"))
) == {"spider1", "spider2"}
- assert (
- self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
- == []
- )
- assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
- assert self.spider_loader.find_by_request(
+ assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == []
+ assert spider_loader.find_by_request(Request("http://spider3.com")) == []
+ assert spider_loader.find_by_request(
Request("http://spider3.com/onlythis")
) == ["spider3"]
def test_load_spider_module(self):
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings({"SPIDER_MODULES": [module]})
- self.spider_loader = SpiderLoader.from_settings(settings)
- assert len(self.spider_loader._spiders) == 1
+ spider_loader = SpiderLoader.from_settings(settings)
+ assert len(spider_loader._spiders) == 1
def test_load_spider_module_multiple(self):
prefix = "tests.test_spiderloader.test_spiders."
module = ",".join(prefix + s for s in ("spider1", "spider2"))
settings = Settings({"SPIDER_MODULES": module})
- self.spider_loader = SpiderLoader.from_settings(settings)
- assert len(self.spider_loader._spiders) == 2
+ spider_loader = SpiderLoader.from_settings(settings)
+ assert len(spider_loader._spiders) == 2
def test_load_base_spider(self):
module = "tests.test_spiderloader.test_spiders.spider0"
settings = Settings({"SPIDER_MODULES": [module]})
- self.spider_loader = SpiderLoader.from_settings(settings)
- assert len(self.spider_loader._spiders) == 0
+ spider_loader = SpiderLoader.from_settings(settings)
+ assert len(spider_loader._spiders) == 0
def test_load_spider_module_from_addons(self):
module = "tests.test_spiderloader.spiders_from_addons.spider0"
@@ -183,27 +184,14 @@ class TestSpiderLoader:
class TestDuplicateSpiderNameLoader:
- def setup_method(self):
- orig_spiders_dir = module_dir / "test_spiders"
- self.tmpdir = Path(mkdtemp())
- self.spiders_dir = self.tmpdir / "test_spiders_xxx"
- _copytree(orig_spiders_dir, self.spiders_dir)
- sys.path.append(str(self.tmpdir))
- self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
-
- def teardown_method(self):
- del sys.modules["test_spiders_xxx"]
- sys.path.remove(str(self.tmpdir))
-
- def test_dupename_warning(self):
+ def test_dupename_warning(self, spider_loader_env):
+ settings, spiders_dir = spider_loader_env
+
# copy 1 spider module so as to have duplicate spider name
- shutil.copyfile(
- self.tmpdir / "test_spiders_xxx" / "spider3.py",
- self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
- )
+ shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")
with warnings.catch_warnings(record=True) as w:
- spider_loader = SpiderLoader.from_settings(self.settings)
+ spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
@@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader:
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
- def test_multiple_dupename_warning(self):
+ def test_multiple_dupename_warning(self, spider_loader_env):
+ settings, spiders_dir = spider_loader_env
# copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name
- shutil.copyfile(
- self.tmpdir / "test_spiders_xxx" / "spider1.py",
- self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
- )
- shutil.copyfile(
- self.tmpdir / "test_spiders_xxx" / "spider2.py",
- self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
- )
+ shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
+ shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
with warnings.catch_warnings(record=True) as w:
- spider_loader = SpiderLoader.from_settings(self.settings)
+ spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
@@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader:
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
+
+
+class CustomSpiderLoader(SpiderLoader):
+ pass
+
+
+def test_custom_spider_loader():
+ settings = Settings(
+ {
+ "SPIDER_LOADER_CLASS": CustomSpiderLoader,
+ }
+ )
+ spider_loader = get_spider_loader(settings)
+ assert isinstance(spider_loader, CustomSpiderLoader)
+
+
+def test_dummy_spider_loader(spider_loader_env):
+ settings, _ = spider_loader_env
+ spider_loader = DummySpiderLoader.from_settings(settings)
+ assert not spider_loader.list()
+ with pytest.raises(KeyError):
+ spider_loader.load("spider1")