Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add SpiderLoaderProtocol. (#6694)

Andrey Rakhmatullin87db3f2fd6f9
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1873c90d3..1ec1e31dc 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -42,7 +42,7 @@ if TYPE_CHECKING:
from collections.abc import Generator, Iterable
from scrapy.logformatter import LogFormatter
- from scrapy.spiderloader import SpiderLoader
+ from scrapy.spiderloader import SpiderLoaderProtocol
from scrapy.statscollectors import StatsCollector
from scrapy.utils.request import RequestFingerprinterProtocol
@@ -282,19 +282,21 @@ class CrawlerRunner:
)
@staticmethod
- def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:
+ def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
- return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))
+ return cast(
+ "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())
+ )
def __init__(self, settings: dict[str, Any] | Settings | None = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
AddonManager.load_pre_crawler_settings(settings)
self.settings: Settings = settings
- self.spider_loader: SpiderLoader = self._get_spider_loader(settings)
+ self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)
self._crawlers: set[Crawler] = set()
self._active: set[Deferred[None]] = set()
self.bootstrap_failed = False
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 210e729a1..f537e0593 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
from zope.interface import implementer
@@ -21,6 +21,23 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
+class SpiderLoaderProtocol(Protocol):
+ @classmethod
+ def from_settings(cls, settings: BaseSettings) -> Self:
+ """Return an instance of the class for the given settings"""
+
+ def load(self, spider_name: str) -> type[Spider]:
+ """Return the Spider class for the given spider name. If the spider
+ name is not found, it must raise a KeyError."""
+
+ def list(self) -> list[str]:
+ """Return a list with the names of all spiders available in the
+ project"""
+
+ def find_by_request(self, request: Request) -> __builtins__.list[str]:
+ """Return the list of spiders names that can handle the given request"""
+
+
@implementer(ISpiderLoader)
class SpiderLoader:
"""
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 5277a292c..74fd0e354 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from scrapy import Request
- from scrapy.spiderloader import SpiderLoader
+ from scrapy.spiderloader import SpiderLoaderProtocol
logger = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:
@overload
def spidercls_for_request(
- spider_loader: SpiderLoader,
+ spider_loader: SpiderLoaderProtocol,
request: Request,
default_spidercls: type[Spider],
log_none: bool = ...,
@@ -74,7 +74,7 @@ def spidercls_for_request(
@overload
def spidercls_for_request(
- spider_loader: SpiderLoader,
+ spider_loader: SpiderLoaderProtocol,
request: Request,
default_spidercls: Literal[None],
log_none: bool = ...,
@@ -84,7 +84,7 @@ def spidercls_for_request(
@overload
def spidercls_for_request(
- spider_loader: SpiderLoader,
+ spider_loader: SpiderLoaderProtocol,
request: Request,
*,
log_none: bool = ...,
@@ -93,7 +93,7 @@ def spidercls_for_request(
def spidercls_for_request(
- spider_loader: SpiderLoader,
+ spider_loader: SpiderLoaderProtocol,
request: Request,
default_spidercls: type[Spider] | None = None,
log_none: bool = False,