Scrapy, a fast high-level web crawling & scraping framework for Python.
Typing improvements for some core components (#5889)
Andrey Rakhmatullinc7730627a0f9
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 1b83c3a8a..dde76a547 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -4,8 +4,9 @@ from datetime import datetimefrom time import timefrom twisted.internet import defer, task+from twisted.internet.defer import Deferred-from scrapy import signals+from scrapy import Request, Spider, signalsfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.resolver import dnscache@@ -86,7 +87,7 @@ class Downloader:self._slot_gc_loop.start(60)self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})- def fetch(self, request, spider):+ def fetch(self, request: Request, spider: Spider) -> Deferred:def _deactivate(response):self.active.remove(request)return response@@ -206,12 +207,12 @@ class Downloader:return dfd.addBoth(finish_transferring)- def close(self):+ def close(self) -> None:self._slot_gc_loop.stop()for slot in self.slots.values():slot.close()- def _slot_gc(self, age=60):+ def _slot_gc(self, age: float = 60) -> None:mintime = time() - agefor key, slot in list(self.slots.items()):if not slot.active and slot.lastseen + slot.delay < mintime:diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex c6738b531..3e5a281b2 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -8,13 +8,16 @@ import loggingimport warningsfrom time import timefrom typing import (+ TYPE_CHECKING,Any,Callable,Generator,Iterable,Iterator,+ List,Optional,Set,+ Type,Union,cast,)@@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloaderfrom scrapy.core.scraper import Scraperfrom scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarningfrom scrapy.http import Request, Response-from scrapy.settings import BaseSettings+from scrapy.logformatter import LogFormatter+from scrapy.settings import BaseSettings, Settings+from scrapy.signalmanager import SignalManagerfrom scrapy.spiders import Spiderfrom scrapy.utils.log import failure_to_exc_info, logformatter_adapterfrom scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.reactor import CallLaterOnce+if TYPE_CHECKING:+ from scrapy.core.scheduler import BaseScheduler+ from scrapy.crawler import Crawler+logger = logging.getLogger(__name__)class Slot:def __init__(self,- start_requests: Iterable,+ start_requests: Iterable[Request],close_if_idle: bool,nextcall: CallLaterOnce,- scheduler,+ scheduler: "BaseScheduler",) -> None:self.closing: Optional[Deferred] = Noneself.inprogress: Set[Request] = set()- self.start_requests: Optional[Iterator] = iter(start_requests)- self.close_if_idle = close_if_idle- self.nextcall = nextcall- self.scheduler = scheduler- self.heartbeat = LoopingCall(nextcall.schedule)+ self.start_requests: Optional[Iterator[Request]] = iter(start_requests)+ self.close_if_idle: bool = close_if_idle+ self.nextcall: CallLaterOnce = nextcall+ self.scheduler: "BaseScheduler" = scheduler+ self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)def add_request(self, request: Request) -> None:self.inprogress.add(request)@@ -75,25 +84,28 @@ class Slot:class ExecutionEngine:- def __init__(self, crawler, spider_closed_callback: Callable) -> None:- self.crawler = crawler- self.settings = crawler.settings- self.signals = crawler.signals- self.logformatter = crawler.logformatter+ def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:+ self.crawler: "Crawler" = crawler+ self.settings: Settings = crawler.settings+ self.signals: SignalManager = crawler.signals+ self.logformatter: LogFormatter = crawler.logformatterself.slot: Optional[Slot] = Noneself.spider: Optional[Spider] = None- self.running = False- self.paused = False- self.scheduler_cls = self._get_scheduler_class(crawler.settings)- downloader_cls = load_object(self.settings["DOWNLOADER"])+ self.running: bool = False+ self.paused: bool = False+ self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(+ crawler.settings+ )+ downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])self.downloader: Downloader = downloader_cls(crawler)self.scraper = Scraper(crawler)- self._spider_closed_callback = spider_closed_callback+ self._spider_closed_callback: Callable = spider_closed_callback+ self.start_time: Optional[float] = None- def _get_scheduler_class(self, settings: BaseSettings) -> type:+ def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:from scrapy.core.scheduler import BaseScheduler- scheduler_cls = load_object(settings["SCHEDULER"])+ scheduler_cls: Type = load_object(settings["SCHEDULER"])if not issubclass(scheduler_cls, BaseScheduler):raise TypeError(f"The provided scheduler class ({settings['SCHEDULER']})"@@ -115,7 +127,7 @@ class ExecutionEngine:"""Gracefully stop the execution engine"""@inlineCallbacks- def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:+ def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)self._closewait.callback(None)@@ -141,7 +153,8 @@ class ExecutionEngine:return self.close_spider(self.spider, reason="shutdown") # will also close downloader- return succeed(self.downloader.close())+ self.downloader.close()+ return succeed(None)def pause(self) -> None:self.paused = True@@ -209,7 +222,7 @@ class ExecutionEngine:extra={"spider": self.spider},))- d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))+ d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type]d.addErrback(lambda f: logger.info("Error while removing request from slot",@@ -339,6 +352,7 @@ class ExecutionEngine:if isinstance(result, Response):if result.request is None:result.request = request+ assert spider is not Nonelogkws = self.logformatter.crawled(result.request, result, spider)if logkws is not None:logger.log(*logformatter_adapter(logkws), extra={"spider": spider})@@ -350,10 +364,12 @@ class ExecutionEngine:)return result- def _on_complete(_):+ def _on_complete(_: Any) -> Any:+ assert self.slot is not Noneself.slot.nextcall.schedule()return _+ assert spider is not Nonedwld = self.downloader.fetch(request, spider)dwld.addCallbacks(_on_success)dwld.addBoth(_on_complete)@@ -362,7 +378,7 @@ class ExecutionEngine:@inlineCallbacksdef open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True- ):+ ) -> Generator[Deferred, Any, None]:if self.slot is not None:raise RuntimeError(f"No free spider slot when opening {spider.name!r}")logger.info("Spider opened", extra={"spider": spider})@@ -471,7 +487,7 @@ class ExecutionEngine:return dfd@property- def open_spiders(self) -> list:+ def open_spiders(self) -> List[Spider]:warnings.warn("ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",category=ScrapyDeprecationWarning,diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 3c46e3a5f..3fb0bbaff 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -2,13 +2,15 @@ import jsonimport loggingfrom abc import abstractmethodfrom pathlib import Path-from typing import Optional, Type, TypeVar+from typing import Any, Optional, Type, TypeVar, castfrom twisted.internet.defer import Deferredfrom scrapy.crawler import Crawler+from scrapy.dupefilters import BaseDupeFilterfrom scrapy.http.request import Requestfrom scrapy.spiders import Spider+from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.job import job_dirfrom scrapy.utils.misc import create_instance, load_object@@ -20,10 +22,10 @@ class BaseSchedulerMeta(type):Metaclass to check scheduler classes against the necessary interface"""- def __instancecheck__(cls, instance):+ def __instancecheck__(cls, instance: Any) -> bool:return cls.__subclasscheck__(type(instance))- def __subclasscheck__(cls, subclass):+ def __subclasscheck__(cls, subclass: type) -> bool:return (hasattr(subclass, "has_pending_requests")and callable(subclass.has_pending_requests)@@ -168,26 +170,26 @@ class Scheduler(BaseScheduler):def __init__(self,- dupefilter,+ dupefilter: BaseDupeFilter,jobdir: Optional[str] = None,dqclass=None,mqclass=None,logunser: bool = False,- stats=None,+ stats: Optional[StatsCollector] = None,pqclass=None,crawler: Optional[Crawler] = None,):- self.df = dupefilter- self.dqdir = self._dqdir(jobdir)+ self.df: BaseDupeFilter = dupefilter+ self.dqdir: Optional[str] = self._dqdir(jobdir)self.pqclass = pqclassself.dqclass = dqclassself.mqclass = mqclass- self.logunser = logunser- self.stats = stats- self.crawler = crawler+ self.logunser: bool = logunser+ self.stats: Optional[StatsCollector] = stats+ self.crawler: Optional[Crawler] = crawler@classmethod- def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:+ def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:"""Factory method, initializes the scheduler with arguments taken from the crawl settings"""@@ -242,6 +244,7 @@ class Scheduler(BaseScheduler):self.df.log(request, self.spider)return Falsedqok = self._dqpush(request)+ assert self.stats is not Noneif dqok:self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)else:@@ -259,7 +262,8 @@ class Scheduler(BaseScheduler):Increment the appropriate stats, such as: ``scheduler/dequeued``,``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``."""- request = self.mqs.pop()+ request: Optional[Request] = self.mqs.pop()+ assert self.stats is not Noneif request is not None:self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)else:@@ -295,6 +299,7 @@ class Scheduler(BaseScheduler):extra={"spider": self.spider},)self.logunser = False+ assert self.stats is not Noneself.stats.inc_value("scheduler/unserializable", spider=self.spider)return Falseelse:@@ -351,7 +356,7 @@ class Scheduler(BaseScheduler):if not path.exists():return []with path.open(encoding="utf-8") as f:- return json.load(f)+ return cast(list, json.load(f))def _write_dqs_state(self, dqdir: str, state: list) -> None:with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 8468b8419..a85f6a661 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -15,6 +15,7 @@ from typing import (Optional,Set,Tuple,+ Type,Union,)@@ -26,6 +27,9 @@ from scrapy import Spider, signalsfrom scrapy.core.spidermw import SpiderMiddlewareManagerfrom scrapy.exceptions import CloseSpider, DropItem, IgnoreRequestfrom scrapy.http import Request, Response+from scrapy.logformatter import LogFormatter+from scrapy.pipelines import ItemPipelineManager+from scrapy.signalmanager import SignalManagerfrom scrapy.utils.defer import (aiter_errback,defer_fail,@@ -96,16 +100,20 @@ class Slot:class Scraper:def __init__(self, crawler: Crawler) -> None:self.slot: Optional[Slot] = None- self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)- itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])- self.itemproc = itemproc_cls.from_crawler(crawler)- self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")- self.crawler = crawler- self.signals = crawler.signals- self.logformatter = crawler.logformatter+ self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(+ crawler+ )+ itemproc_cls: Type[ItemPipelineManager] = load_object(+ crawler.settings["ITEM_PROCESSOR"]+ )+ self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)+ self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")+ self.crawler: Crawler = crawler+ self.signals: SignalManager = crawler.signals+ self.logformatter: LogFormatter = crawler.logformatter@inlineCallbacks- def open_spider(self, spider: Spider):+ def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:"""Open the given spider for scraping and allocate resources for it"""self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))yield self.itemproc.open_spider(spider)@@ -135,7 +143,8 @@ class Scraper:raise RuntimeError("Scraper slot not assigned")dfd = self.slot.add_response_request(result, request)- def finish_scraping(_):+ def finish_scraping(_: Any) -> Any:+ assert self.slot is not Noneself.slot.finish_response(result, request)self._check_if_closing(spider)self._scrape_next(spider)@@ -205,9 +214,9 @@ class Scraper:else: # result is a Failure# TODO: properly type adding this attribute to a Failureresult.request = request # type: ignore[attr-defined]- warn_on_generator_with_return_value(spider, request.errback)dfd = defer_fail(result)if request.errback:+ warn_on_generator_with_return_value(spider, request.errback)dfd.addErrback(request.errback)return dfd.addCallback(iterate_spider_output)@@ -338,7 +347,7 @@ class Scraper:def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider- ) -> None:+ ) -> Deferred:"""ItemProcessor finished for the given ``item`` and returned ``output``"""assert self.slot is not None # typingself.slot.itemproc_size -= 1diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex c21985b18..dcf1a6dbc 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -13,6 +13,8 @@ from typing import (Callable,Generator,Iterable,+ List,+ Optional,Tuple,Union,cast,@@ -25,6 +27,7 @@ from scrapy import Request, Spiderfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Responsefrom scrapy.middleware import MiddlewareManager+from scrapy.settings import BaseSettingsfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import (@@ -41,22 +44,22 @@ logger = logging.getLogger(__name__)ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]-def _isiterable(o) -> bool:+def _isiterable(o: Any) -> bool:return isinstance(o, (Iterable, AsyncIterable))class SpiderMiddlewareManager(MiddlewareManager):component_name = "spider middleware"- def __init__(self, *middlewares):+ def __init__(self, *middlewares: Any):super().__init__(*middlewares)self.downgrade_warning_done = False@classmethod- def _get_mwlist_from_settings(cls, settings):+ def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]:return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))- def _add_middleware(self, mw):+ def _add_middleware(self, mw: Any) -> None:super()._add_middleware(mw)if hasattr(mw, "process_spider_input"):self.methods["process_spider_input"].append(mw.process_spider_input)@@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager):exception_processor_index: int,recover_to: Union[MutableChain, MutableAsyncChain],) -> Union[Generator, AsyncGenerator]:- def process_sync(iterable: Iterable):+ def process_sync(iterable: Iterable) -> Generator:try:for r in iterable:yield r@@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager):raiserecover_to.extend(exception_result)- async def process_async(iterable: AsyncIterable):+ async def process_async(iterable: AsyncIterable) -> AsyncGenerator:try:async for r in iterable:yield r@@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager):if isinstance(recovered, AsyncIterable):recovered_collected = await collect_asyncgen(recovered)recovered = MutableChain(recovered_collected)- return MutableChain(result, recovered) # type: ignore[arg-type]+ return MutableChain(result, recovered)def scrape_response(self,@@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager):)return dfd- def process_start_requests(self, start_requests, spider: Spider) -> Deferred:+ def process_start_requests(+ self, start_requests: Iterable[Request], spider: Spider+ ) -> Deferred:return self._process_chain("process_start_requests", start_requests, spider)# This method is only needed until _async compatibility methods are removed.@@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager):def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:- normal_method = getattr(mw, methodname, None)+ normal_method: Optional[Callable] = getattr(mw, methodname, None)methodname_async = methodname + "_async"- async_method = getattr(mw, methodname_async, None)+ async_method: Optional[Callable] = getattr(mw, methodname_async, None)if not async_method:return normal_methodif not normal_method:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 397817d6f..256f6e2c5 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -4,7 +4,7 @@ import loggingimport pprintimport signalimport warnings-from typing import TYPE_CHECKING, Optional+from typing import TYPE_CHECKING, Optional, Type, Unionfrom twisted.internet import deferfrom zope.interface.exceptions import DoesNotImplement@@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEnginefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManagerfrom scrapy.interfaces import ISpiderLoader+from scrapy.logformatter import LogFormatterfrom scrapy.settings import Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager+from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.log import (LogCounterHandler,configure_logging,@@ -49,20 +51,25 @@ logger = logging.getLogger(__name__)class Crawler:- def __init__(self, spidercls, settings=None, init_reactor: bool = False):+ def __init__(+ self,+ spidercls: Type[Spider],+ settings: Union[None, dict, Settings] = None,+ init_reactor: bool = False,+ ):if isinstance(spidercls, Spider):raise ValueError("The spidercls argument must be a class, not an object")if isinstance(settings, dict) or settings is None:settings = Settings(settings)- self.spidercls = spidercls- self.settings = settings.copy()+ self.spidercls: Type[Spider] = spidercls+ self.settings: Settings = settings.copy()self.spidercls.update_settings(self.settings)- self.signals = SignalManager(self)+ self.signals: SignalManager = SignalManager(self)- self.stats = load_object(self.settings["STATS_CLASS"])(self)+ self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))logging.root.addHandler(handler)@@ -80,8 +87,8 @@ class Crawler:self.__remove_handler = lambda: logging.root.removeHandler(handler)self.signals.connect(self.__remove_handler, signals.engine_stopped)- lf_cls = load_object(self.settings["LOG_FORMATTER"])- self.logformatter = lf_cls.from_crawler(self)+ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])+ self.logformatter: LogFormatter = lf_cls.from_crawler(self)self.request_fingerprinter: RequestFingerprinter = create_instance(load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),@@ -89,8 +96,8 @@ class Crawler:crawler=self,)- reactor_class = self.settings["TWISTED_REACTOR"]- event_loop = self.settings["ASYNCIO_EVENT_LOOP"]+ reactor_class: str = self.settings["TWISTED_REACTOR"]+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]if init_reactor:# this needs to be done after the spider settings are merged,# but before something imports twisted.internet.reactor@@ -104,11 +111,11 @@ class Crawler:if is_asyncio_reactor_installed() and event_loop:verify_installed_asyncio_event_loop(event_loop)- self.extensions = ExtensionManager.from_crawler(self)+ self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)self.settings.freeze()- self.crawling = False- self.spider = None+ self.crawling: bool = False+ self.spider: Optional[Spider] = Noneself.engine: Optional[ExecutionEngine] = None@defer.inlineCallbacksdiff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 7f202b8b8..fedd02805 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -39,7 +39,7 @@ class DontCloseSpider(Exception):class CloseSpider(Exception):"""Raise this from callbacks to request the spider to be closed"""- def __init__(self, reason="cancelled"):+ def __init__(self, reason: str = "cancelled"):super().__init__()self.reason = reasondiff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 560006c95..7cb379b46 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -1,8 +1,11 @@import loggingimport os+from typing import Any, Dict, Optional, Unionfrom twisted.python.failure import Failure+from scrapy import Request, Spider+from scrapy.http import Responsefrom scrapy.utils.request import referer_strSCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"@@ -52,7 +55,7 @@ class LogFormatter:}"""- def crawled(self, request, response, spider):+ def crawled(self, request: Request, response: Response, spider: Spider) -> dict:"""Logs a message when the crawler finds a webpage."""request_flags = f" {str(request.flags)}" if request.flags else ""response_flags = f" {str(response.flags)}" if response.flags else ""@@ -70,8 +73,11 @@ class LogFormatter:},}- def scraped(self, item, response, spider):+ def scraped(+ self, item: Any, response: Union[Response, Failure], spider: Spider+ ) -> dict:"""Logs a message when an item is scraped by a spider."""+ src: Anyif isinstance(response, Failure):src = response.getErrorMessage()else:@@ -85,7 +91,9 @@ class LogFormatter:},}- def dropped(self, item, exception, response, spider):+ def dropped(+ self, item: Any, exception: BaseException, response: Response, spider: Spider+ ) -> dict:"""Logs a message when an item is dropped while it is passing through the item pipeline."""return {"level": logging.WARNING,@@ -96,7 +104,9 @@ class LogFormatter:},}- def item_error(self, item, exception, response, spider):+ def item_error(+ self, item: Any, exception, response: Response, spider: Spider+ ) -> dict:"""Logs a message when an item causes an error while it is passingthrough the item pipeline.@@ -110,7 +120,9 @@ class LogFormatter:},}- def spider_error(self, failure, request, response, spider):+ def spider_error(+ self, failure: Failure, request: Request, response: Response, spider: Spider+ ) -> dict:"""Logs an error message from a spider... versionadded:: 2.0@@ -124,13 +136,19 @@ class LogFormatter:},}- def download_error(self, failure, request, spider, errmsg=None):+ def download_error(+ self,+ failure: Failure,+ request: Request,+ spider: Spider,+ errmsg: Optional[str] = None,+ ) -> dict:"""Logs a download error message from a spider (typically coming fromthe engine)... versionadded:: 2.0"""- args = {"request": request}+ args: Dict[str, Any] = {"request": request}if errmsg:msg = DOWNLOADERRORMSG_LONGargs["errmsg"] = errmsgdiff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex f82d722fa..03e92b565 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -1,7 +1,7 @@import loggingimport pprintfrom collections import defaultdict, deque-from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast+from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, castfrom twisted.internet.defer import Deferred@@ -30,7 +30,7 @@ class MiddlewareManager:self._add_middleware(mw)@classmethod- def _get_mwlist_from_settings(cls, settings: Settings) -> list:+ def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:raise NotImplementedError@classmethod@@ -67,17 +67,17 @@ class MiddlewareManager:def from_crawler(cls, crawler):return cls.from_settings(crawler.settings, crawler)- def _add_middleware(self, mw) -> None:+ def _add_middleware(self, mw: Any) -> None:if hasattr(mw, "open_spider"):self.methods["open_spider"].append(mw.open_spider)if hasattr(mw, "close_spider"):self.methods["close_spider"].appendleft(mw.close_spider)- def _process_parallel(self, methodname: str, obj, *args) -> Deferred:+ def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:methods = cast(Iterable[Callable], self.methods[methodname])return process_parallel(methods, obj, *args)- def _process_chain(self, methodname: str, obj, *args) -> Deferred:+ def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:methods = cast(Iterable[Callable], self.methods[methodname])return process_chain(methods, obj, *args)diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex df574a0a1..c97d71fb6 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -3,7 +3,11 @@ Item pipelineSee documentation in docs/item-pipeline.rst"""+from typing import Any, List+from twisted.internet.defer import Deferred++from scrapy import Spiderfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import deferred_f_from_coro_f@@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager):component_name = "item pipeline"@classmethod- def _get_mwlist_from_settings(cls, settings):+ def _get_mwlist_from_settings(cls, settings) -> List[Any]:return build_component_list(settings.getwithbase("ITEM_PIPELINES"))- def _add_middleware(self, pipe):+ def _add_middleware(self, pipe: Any) -> None:super()._add_middleware(pipe)if hasattr(pipe, "process_item"):self.methods["process_item"].append(deferred_f_from_coro_f(pipe.process_item))- def process_item(self, item, spider):+ def process_item(self, item: Any, spider: Spider) -> Deferred:return self._process_chain("process_item", item, spider)diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.pyindex d7e3bce91..f6df191d8 100644--- a/scrapy/signalmanager.py+++ b/scrapy/signalmanager.py@@ -1,13 +1,16 @@+from typing import Any, List, Tuple+from pydispatch import dispatcher+from twisted.internet.defer import Deferredfrom scrapy.utils import signal as _signalclass SignalManager:- def __init__(self, sender=dispatcher.Anonymous):- self.sender = sender+ def __init__(self, sender: Any = dispatcher.Anonymous):+ self.sender: Any = sender- def connect(self, receiver, signal, **kwargs):+ def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:"""Connect a receiver function to a signal.@@ -22,18 +25,18 @@ class SignalManager::type signal: object"""kwargs.setdefault("sender", self.sender)- return dispatcher.connect(receiver, signal, **kwargs)+ dispatcher.connect(receiver, signal, **kwargs)- def disconnect(self, receiver, signal, **kwargs):+ def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:"""Disconnect a receiver function from a signal. This has theopposite effect of the :meth:`connect` method, and the argumentsare the same."""kwargs.setdefault("sender", self.sender)- return dispatcher.disconnect(receiver, signal, **kwargs)+ dispatcher.disconnect(receiver, signal, **kwargs)- def send_catch_log(self, signal, **kwargs):+ def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]:"""Send a signal, catch exceptions and log them.@@ -43,7 +46,7 @@ class SignalManager:kwargs.setdefault("sender", self.sender)return _signal.send_catch_log(signal, **kwargs)- def send_catch_log_deferred(self, signal, **kwargs):+ def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:"""Like :meth:`send_catch_log` but supports returning:class:`~twisted.internet.defer.Deferred` objects from signal handlers.@@ -57,7 +60,7 @@ class SignalManager:kwargs.setdefault("sender", self.sender)return _signal.send_catch_log_deferred(signal, **kwargs)- def disconnect_all(self, signal, **kwargs):+ def disconnect_all(self, signal: Any, **kwargs: Any) -> None:"""Disconnect all receivers from the given signal.@@ -65,4 +68,4 @@ class SignalManager::type signal: object"""kwargs.setdefault("sender", self.sender)- return _signal.disconnect_all(signal, **kwargs)+ _signal.disconnect_all(signal, **kwargs)diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.pyindex dd3c32737..15193aac5 100644--- a/scrapy/statscollectors.py+++ b/scrapy/statscollectors.py@@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats"""import loggingimport pprint+from typing import TYPE_CHECKING, Any, Dict, Optional++from scrapy import Spider++if TYPE_CHECKING:+ from scrapy.crawler import Crawlerlogger = logging.getLogger(__name__)+StatsT = Dict[str, Any]++class StatsCollector:- def __init__(self, crawler):- self._dump = crawler.settings.getbool("STATS_DUMP")- self._stats = {}+ def __init__(self, crawler: "Crawler"):+ self._dump: bool = crawler.settings.getbool("STATS_DUMP")+ self._stats: StatsT = {}- def get_value(self, key, default=None, spider=None):+ def get_value(+ self, key: str, default: Any = None, spider: Optional[Spider] = None+ ) -> Any:return self._stats.get(key, default)- def get_stats(self, spider=None):+ def get_stats(self, spider: Optional[Spider] = None) -> StatsT:return self._stats- def set_value(self, key, value, spider=None):+ def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:self._stats[key] = value- def set_stats(self, stats, spider=None):+ def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:self._stats = stats- def inc_value(self, key, count=1, start=0, spider=None):+ def inc_value(+ self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None+ ) -> None:d = self._statsd[key] = d.setdefault(key, start) + count- def max_value(self, key, value, spider=None):+ def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:self._stats[key] = max(self._stats.setdefault(key, value), value)- def min_value(self, key, value, spider=None):+ def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:self._stats[key] = min(self._stats.setdefault(key, value), value)- def clear_stats(self, spider=None):+ def clear_stats(self, spider: Optional[Spider] = None) -> None:self._stats.clear()- def open_spider(self, spider):+ def open_spider(self, spider: Spider) -> None:pass- def close_spider(self, spider, reason):+ def close_spider(self, spider: Spider, reason: str) -> None:if self._dump:logger.info("Dumping Scrapy stats:\n" + pprint.pformat(self._stats),@@ -48,34 +61,38 @@ class StatsCollector:)self._persist_stats(self._stats, spider)- def _persist_stats(self, stats, spider):+ def _persist_stats(self, stats: StatsT, spider: Spider) -> None:passclass MemoryStatsCollector(StatsCollector):- def __init__(self, crawler):+ def __init__(self, crawler: "Crawler"):super().__init__(crawler)- self.spider_stats = {}+ self.spider_stats: Dict[str, StatsT] = {}- def _persist_stats(self, stats, spider):+ def _persist_stats(self, stats: StatsT, spider: Spider) -> None:self.spider_stats[spider.name] = statsclass DummyStatsCollector(StatsCollector):- def get_value(self, key, default=None, spider=None):+ def get_value(+ self, key: str, default: Any = None, spider: Optional[Spider] = None+ ) -> Any:return default- def set_value(self, key, value, spider=None):+ def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:pass- def set_stats(self, stats, spider=None):+ def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:pass- def inc_value(self, key, count=1, start=0, spider=None):+ def inc_value(+ self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None+ ) -> None:pass- def max_value(self, key, value, spider=None):+ def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:pass- def min_value(self, key, value, spider=None):+ def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:passdiff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 6ae27dc29..2ce4725f4 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -2,6 +2,7 @@ import loggingimport sysimport warningsfrom logging.config import dictConfig+from typing import Tuplefrom twisted.python import log as twisted_logfrom twisted.python.failure import Failure@@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versionslogger = logging.getLogger(__name__)-def failure_to_exc_info(failure):+def failure_to_exc_info(failure: Failure):"""Extract exc_info from Failure instances"""if isinstance(failure, Failure):return (failure.type, failure.value, failure.getTracebackObject())@@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler):self.crawler.stats.inc_value(sname)-def logformatter_adapter(logkws):+def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:"""Helper that takes the dictionary output from the methods in LogFormatterand adapts it into a tuple of positional arguments for logger.log calls,diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex f9f9c0d5b..d861c9ab6 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -10,6 +10,7 @@ from contextlib import contextmanagerfrom functools import partialfrom importlib import import_modulefrom pkgutil import iter_modules+from typing import TYPE_CHECKING, Any, Callable, Unionfrom w3lib.html import replace_entities@@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCachefrom scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.python import flatten, to_unicode+if TYPE_CHECKING:+ from scrapy import Spider++_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes@@ -34,7 +39,7 @@ def arg_to_iter(arg):return [arg]-def load_object(path):+def load_object(path: Union[str, Callable]) -> Any:"""Load an object given its absolute object path, and return it.The object can be the import path of a class, function, variable or an@@ -249,7 +254,7 @@ def is_generator_with_return_value(callable):return _generator_callbacks_cache[callable]-def warn_on_generator_with_return_value(spider, callable):+def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:"""Logs a warning if a callable is a generator function and includesa 'return' statement with a value different than Nonediff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 7f67d036a..f1b9239e6 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -1,9 +1,11 @@import asyncioimport sysfrom contextlib import suppress+from typing import Any, Callable, Dict, Optional, Sequencefrom warnings import catch_warnings, filterwarnings, warnfrom twisted.internet import asyncioreactor, error+from twisted.internet.base import DelayedCallfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import load_object@@ -34,23 +36,23 @@ class CallLaterOnce:it hasn't been already scheduled since the last time it ran."""- def __init__(self, func, *a, **kw):- self._func = func- self._a = a- self._kw = kw- self._call = None+ def __init__(self, func: Callable, *a: Any, **kw: Any):+ self._func: Callable = func+ self._a: Sequence[Any] = a+ self._kw: Dict[str, Any] = kw+ self._call: Optional[DelayedCall] = None- def schedule(self, delay=0):+ def schedule(self, delay: float = 0) -> None:from twisted.internet import reactorif self._call is None:self._call = reactor.callLater(delay, self)- def cancel(self):+ def cancel(self) -> None:if self._call:self._call.cancel()- def __call__(self):+ def __call__(self) -> Any:self._call = Nonereturn self._func(*self._a, **self._kw)diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex b95786d35..9e7ddd827 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -1,6 +1,8 @@"""Helper functions for working with signals"""import collections.abcimport logging+from typing import Any as TypingAny+from typing import List, Tuplefrom pydispatch.dispatcher import (Anonymous,@@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_infologger = logging.getLogger(__name__)-def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):+def send_catch_log(+ signal=Any, sender=Anonymous, *arguments, **named+) -> List[Tuple[TypingAny, TypingAny]]:"""Like pydispatcher.robust.sendRobust but it also logs errors and returnsFailures instead of exceptions."""@@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):)dont_log += (StopDownload,)spider = named.get("spider", None)- responses = []+ responses: List[Tuple[TypingAny, TypingAny]] = []for receiver in liveReceivers(getAllReceivers(sender, signal)):+ result: TypingAnytry:response = robustApply(receiver, signal=signal, sender=sender, *arguments, **nameddiff --git a/tox.ini b/tox.iniindex 5a9d9cf29..d96a278ea 100644--- a/tox.ini+++ b/tox.ini@@ -37,13 +37,13 @@ install_command =[testenv:typing]basepython = python3deps =- mypy==1.0.1+ mypy==1.2.0types-attrs==19.1.0- types-lxml==2023.2.11- types-Pillow==9.4.0.16- types-Pygments==2.14.0.5- types-pyOpenSSL==23.0.0.4- types-setuptools==67.4.0.1+ types-lxml==2023.3.28+ types-Pillow==9.4.0.19+ types-Pygments==2.14.0.7+ types-pyOpenSSL==23.1.0.1+ types-setuptools==67.6.0.7commands =mypy --show-error-codes {posargs: scrapy tests}