Scrapy, a fast high-level web crawling & scraping framework for Python.
Misc typing improvements. (#6494)
Andrey Rakhmatullinf65e64a7243d
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex c7946605b..1ce155da7 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environclass TextTestResult(_TextTestResult):def printSummary(self, start: float, stop: float) -> None:write = self.stream.write- # _WritelnDecorator isn't implemented in typeshed yet- writeln = self.stream.writeln # type: ignore[attr-defined]+ writeln = self.stream.writelnrun = self.testsRunplural = "s" if run != 1 else ""@@ -84,7 +83,7 @@ class Command(ScrapyCommand):with set_environ(SCRAPY_CHECK="true"):for spidername in args or spider_loader.list():spidercls = spider_loader.load(spidername)- spidercls.start_requests = lambda s: conman.from_spider(s, result)+ spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value]tested_methods = conman.tested_methods_from_spidercls(spidercls)if opts.list:diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex a9b7a6eee..2ac281212 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -4,7 +4,7 @@ import shutilimport stringfrom importlib import import_modulefrom pathlib import Path-from typing import Optional, Union, cast+from typing import Any, Optional, Union, castfrom urllib.parse import urlparseimport scrapy@@ -122,7 +122,7 @@ class Command(ScrapyCommand):name: str,url: str,template_name: str,- ):+ ) -> dict[str, Any]:capitalized_module = "".join(s.capitalize() for s in module.split("_"))return {"project_name": self.settings.get("BOT_NAME"),diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex bd1fad14b..ff2bb8ab9 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -38,9 +38,10 @@ _T = TypeVar("_T")class Command(BaseRunSpiderCommand):requires_project = True- spider = None+ spider: Optional[Spider] = Noneitems: dict[int, list[Any]] = {}requests: dict[int, list[Request]] = {}+ spidercls: Optional[type[Spider]]first_response = None@@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand):yield self.prepare_request(spider, Request(url), opts)if self.spidercls:- self.spidercls.start_requests = _start_requests+ self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign]def start_parsing(self, url: str, opts: argparse.Namespace) -> None:assert self.crawler_process+ assert self.spiderclsself.crawler_process.crawl(self.spidercls, **opts.spargs)self.pcrawler = list(self.crawler_process.crawlers)[0]self.crawler_process.start()diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex bb09d066f..f3d74eccf 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -100,7 +100,7 @@ class ExecutionEngine:)downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])self.downloader: Downloader = downloader_cls(crawler)- self.scraper = Scraper(crawler)+ self.scraper: Scraper = Scraper(crawler)self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (spider_closed_callback)@@ -325,7 +325,7 @@ class ExecutionEngine:raise RuntimeError(f"No open spider to crawl: {request}")d: Deferred[Union[Response, Request]] = self._download(request)# Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type- d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type]+ d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload]return d2def _downloaded(diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 29d7cb0c8..71a0d6aeb 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -55,7 +55,7 @@ class Slot:MIN_RESPONSE_SIZE = 1024def __init__(self, max_active_size: int = 5000000):- self.max_active_size = max_active_size+ self.max_active_size: int = max_active_sizeself.queue: deque[QueueTuple] = deque()self.active: set[Request] = set()self.active_size: int = 0@@ -316,7 +316,9 @@ class Scraper:)return None- def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:+ def start_itemproc(+ self, item: Any, *, response: Optional[Response]+ ) -> Deferred[Any]:"""Send *item* to the item pipelines for processing.*response* is the source of the item data. If the item does not comediff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 223e4192e..3c8513042 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager):def _process_spider_input(self,- scrape_func: ScrapeFunc,+ scrape_func: ScrapeFunc[_T],response: Response,request: Request,spider: Spider,@@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager):def scrape_response(self,- scrape_func: ScrapeFunc,+ scrape_func: ScrapeFunc[_T],response: Response,request: Request,spider: Spider,diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex b0a4932e1..e75ef52ac 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -42,8 +42,9 @@ from scrapy.utils.reactor import ()if TYPE_CHECKING:- from collections.abc import Generator+ from collections.abc import Generator, Iterable+ from scrapy.spiderloader import SpiderLoaderfrom scrapy.utils.request import RequestFingerprinter@@ -178,16 +179,18 @@ class Crawler:yield maybeDeferred(self.engine.stop)@staticmethod- def _get_component(component_class, components):+ def _get_component(+ component_class: type[_T], components: Iterable[Any]+ ) -> Optional[_T]:for component in components:if isinstance(component, component_class):return componentreturn None- def get_addon(self, cls):+ def get_addon(self, cls: type[_T]) -> Optional[_T]:return self._get_component(cls, self.addons.addons)- def get_downloader_middleware(self, cls):+ def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]:if not self.engine:raise RuntimeError("Crawler.get_downloader_middleware() can only be called after "@@ -195,7 +198,7 @@ class Crawler:)return self._get_component(cls, self.engine.downloader.middleware.middlewares)- def get_extension(self, cls):+ def get_extension(self, cls: type[_T]) -> Optional[_T]:if not self.extensions:raise RuntimeError("Crawler.get_extension() can only be called after the "@@ -203,7 +206,7 @@ class Crawler:)return self._get_component(cls, self.extensions.middlewares)- def get_item_pipeline(self, cls):+ def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]:if not self.engine:raise RuntimeError("Crawler.get_item_pipeline() can only be called after the "@@ -211,7 +214,7 @@ class Crawler:)return self._get_component(cls, self.engine.scraper.itemproc.middlewares)- def get_spider_middleware(self, cls):+ def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]:if not self.engine:raise RuntimeError("Crawler.get_spider_middleware() can only be called after the "@@ -240,18 +243,18 @@ class CrawlerRunner:)@staticmethod- def _get_spider_loader(settings: BaseSettings):+ def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:"""Get SpiderLoader instance from settings"""cls_path = settings.get("SPIDER_LOADER_CLASS")loader_cls = load_object(cls_path)verifyClass(ISpiderLoader, loader_cls)- return loader_cls.from_settings(settings.frozencopy())+ return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))def __init__(self, settings: Union[dict[str, Any], Settings, None] = None):if isinstance(settings, dict) or settings is None:settings = Settings(settings)- self.settings = settings- self.spider_loader = self._get_spider_loader(settings)+ self.settings: Settings = settings+ self.spider_loader: SpiderLoader = self._get_spider_loader(settings)self._crawlers: set[Crawler] = set()self._active: set[Deferred[None]] = set()self.bootstrap_failed = False@@ -329,8 +332,7 @@ class CrawlerRunner:def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler:if isinstance(spidercls, str):spidercls = self.spider_loader.load(spidercls)- # temporary cast until self.spider_loader is typed- return Crawler(cast(type[Spider], spidercls), self.settings)+ return Crawler(spidercls, self.settings)def stop(self) -> Deferred[Any]:"""@@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner):super().__init__(settings)configure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)- self._initialized_reactor = False+ self._initialized_reactor: bool = Falsedef _signal_shutdown(self, signum: int, _: Any) -> None:from twisted.internet import reactor@@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner):init_reactor = not self._initialized_reactorself._initialized_reactor = True# temporary cast until self.spider_loader is typed- return Crawler(- cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor- )+ return Crawler(spidercls, self.settings, init_reactor=init_reactor)def start(self, stop_after_crawl: bool = True, install_signal_handlers: bool = Truediff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex b0cede97d..d913ca25d 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -88,7 +88,7 @@ class HttpCompressionMiddleware:crawler.signals.connect(mw.open_spider, signals.spider_opened)return mw- def open_spider(self, spider):+ def open_spider(self, spider: Spider) -> None:if hasattr(spider, "download_maxsize"):self._max_size = spider.download_maxsizeif hasattr(spider, "download_warnsize"):diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 421c58e68..81ba009d6 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -67,7 +67,7 @@ class RobotsTxtMiddleware:if request.url.startswith("data:") or request.url.startswith("file:"):return Noned: Deferred[Optional[RobotParser]] = maybeDeferred(- self.robot_parser, request, spider # type: ignore[arg-type]+ self.robot_parser, request, spider # type: ignore[call-overload])d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)return d2diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex b1001dabb..7bfcbe6f3 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -578,7 +578,7 @@ class FeedExporter:return Nonelogmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"- d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type]+ d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload]d.addCallback(self._handle_store_success, logmsg, spider, type(slot.storage).__name__diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 9314856c1..32e9ffe7c 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline):path = self.file_path(request, info=info, item=item)# maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type- dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]+ dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)dfd2.addErrback(lambda _: None)dfd2.addErrback(diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 3a0dee8f1..aeacadb1c 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -305,7 +305,11 @@ def process_parallel(dfds, fireOnOneErrback=True, consumeErrors=True)d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r])- d2.addErrback(lambda f: f.value.subFailure)++ def eb(failure: Failure) -> Failure:+ return failure.value.subFailure++ d2.addErrback(eb)return d2diff --git a/tox.ini b/tox.iniindex dad15c6ab..79f72a0f2 100644--- a/tox.ini+++ b/tox.ini@@ -46,12 +46,12 @@ install_command =[testenv:typing]basepython = python3deps =- mypy==1.11.1+ mypy==1.12.0typing-extensions==4.12.2- types-lxml==2024.8.7+ types-lxml==2024.9.16types-Pygments==2.18.0.20240506- botocore-stubs==1.34.158- boto3-stubs[s3]==1.34.158+ botocore-stubs==1.35.39+ boto3-stubs[s3]==1.35.39attrs >= 18.2.0Pillow >= 10.3.0pyOpenSSL >= 24.2.1