Scrapy, a fast high-level web crawling & scraping framework for Python.
Add SpiderLoaderProtocol. (#6694)
Andrey Rakhmatullin87db3f2fd6f9
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 1873c90d3..1ec1e31dc 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -42,7 +42,7 @@ if TYPE_CHECKING:from collections.abc import Generator, Iterablefrom scrapy.logformatter import LogFormatter- from scrapy.spiderloader import SpiderLoader+ from scrapy.spiderloader import SpiderLoaderProtocolfrom scrapy.statscollectors import StatsCollectorfrom scrapy.utils.request import RequestFingerprinterProtocol@@ -282,19 +282,21 @@ class CrawlerRunner:)@staticmethod- def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:+ def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:"""Get SpiderLoader instance from settings"""cls_path = settings.get("SPIDER_LOADER_CLASS")loader_cls = load_object(cls_path)verifyClass(ISpiderLoader, loader_cls)- return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))+ return cast(+ "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())+ )def __init__(self, settings: dict[str, Any] | Settings | None = None):if isinstance(settings, dict) or settings is None:settings = Settings(settings)AddonManager.load_pre_crawler_settings(settings)self.settings: Settings = settings- self.spider_loader: SpiderLoader = self._get_spider_loader(settings)+ self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)self._crawlers: set[Crawler] = set()self._active: set[Deferred[None]] = set()self.bootstrap_failed = Falsediff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex 210e729a1..f537e0593 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport tracebackimport warningsfrom collections import defaultdict-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Protocolfrom zope.interface import implementer@@ -21,6 +21,23 @@ if TYPE_CHECKING:from scrapy.settings import BaseSettings+class SpiderLoaderProtocol(Protocol):+ @classmethod+ def from_settings(cls, settings: BaseSettings) -> Self:+ """Return an instance of the class for the given settings"""++ def load(self, spider_name: str) -> type[Spider]:+ """Return the Spider class for the given spider name. If the spider+ name is not found, it must raise a KeyError."""++ def list(self) -> list[str]:+ """Return a list with the names of all spiders available in the+ project"""++ def find_by_request(self, request: Request) -> __builtins__.list[str]:+ """Return the list of spiders names that can handle the given request"""++@implementer(ISpiderLoader)class SpiderLoader:"""diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 5277a292c..74fd0e354 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -15,7 +15,7 @@ if TYPE_CHECKING:from twisted.internet.defer import Deferredfrom scrapy import Request- from scrapy.spiderloader import SpiderLoader+ from scrapy.spiderloader import SpiderLoaderProtocollogger = logging.getLogger(__name__)@@ -64,7 +64,7 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:@overloaddef spidercls_for_request(- spider_loader: SpiderLoader,+ spider_loader: SpiderLoaderProtocol,request: Request,default_spidercls: type[Spider],log_none: bool = ...,@@ -74,7 +74,7 @@ def spidercls_for_request(@overloaddef spidercls_for_request(- spider_loader: SpiderLoader,+ spider_loader: SpiderLoaderProtocol,request: Request,default_spidercls: Literal[None],log_none: bool = ...,@@ -84,7 +84,7 @@ def spidercls_for_request(@overloaddef spidercls_for_request(- spider_loader: SpiderLoader,+ spider_loader: SpiderLoaderProtocol,request: Request,*,log_none: bool = ...,@@ -93,7 +93,7 @@ def spidercls_for_request(def spidercls_for_request(- spider_loader: SpiderLoader,+ spider_loader: SpiderLoaderProtocol,request: Request,default_spidercls: type[Spider] | None = None,log_none: bool = False,