Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Typing improvements for some core components (#5889)

Andrey Rakhmatullinc7730627a0f9
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 1b83c3a8a..dde76a547 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,8 +4,9 @@ from datetime import datetime
from time import time
from twisted.internet import defer, task
+from twisted.internet.defer import Deferred
-from scrapy import signals
+from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.resolver import dnscache
@@ -86,7 +87,7 @@ class Downloader:
self._slot_gc_loop.start(60)
self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
- def fetch(self, request, spider):
+ def fetch(self, request: Request, spider: Spider) -> Deferred:
def _deactivate(response):
self.active.remove(request)
return response
@@ -206,12 +207,12 @@ class Downloader:
return dfd.addBoth(finish_transferring)
- def close(self):
+ def close(self) -> None:
self._slot_gc_loop.stop()
for slot in self.slots.values():
slot.close()
- def _slot_gc(self, age=60):
+ def _slot_gc(self, age: float = 60) -> None:
mintime = time() - age
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index c6738b531..3e5a281b2 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -8,13 +8,16 @@ import logging
import warnings
from time import time
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Generator,
Iterable,
Iterator,
+ List,
Optional,
Set,
+ Type,
Union,
cast,
)
@@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
+from scrapy.logformatter import LogFormatter
+from scrapy.settings import BaseSettings, Settings
+from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.reactor import CallLaterOnce
+if TYPE_CHECKING:
+ from scrapy.core.scheduler import BaseScheduler
+ from scrapy.crawler import Crawler
+
logger = logging.getLogger(__name__)
class Slot:
def __init__(
self,
- start_requests: Iterable,
+ start_requests: Iterable[Request],
close_if_idle: bool,
nextcall: CallLaterOnce,
- scheduler,
+ scheduler: "BaseScheduler",
) -> None:
self.closing: Optional[Deferred] = None
self.inprogress: Set[Request] = set()
- self.start_requests: Optional[Iterator] = iter(start_requests)
- self.close_if_idle = close_if_idle
- self.nextcall = nextcall
- self.scheduler = scheduler
- self.heartbeat = LoopingCall(nextcall.schedule)
+ self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
+ self.close_if_idle: bool = close_if_idle
+ self.nextcall: CallLaterOnce = nextcall
+ self.scheduler: "BaseScheduler" = scheduler
+ self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
def add_request(self, request: Request) -> None:
self.inprogress.add(request)
@@ -75,25 +84,28 @@ class Slot:
class ExecutionEngine:
- def __init__(self, crawler, spider_closed_callback: Callable) -> None:
- self.crawler = crawler
- self.settings = crawler.settings
- self.signals = crawler.signals
- self.logformatter = crawler.logformatter
+ def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
+ self.crawler: "Crawler" = crawler
+ self.settings: Settings = crawler.settings
+ self.signals: SignalManager = crawler.signals
+ self.logformatter: LogFormatter = crawler.logformatter
self.slot: Optional[Slot] = None
self.spider: Optional[Spider] = None
- self.running = False
- self.paused = False
- self.scheduler_cls = self._get_scheduler_class(crawler.settings)
- downloader_cls = load_object(self.settings["DOWNLOADER"])
+ self.running: bool = False
+ self.paused: bool = False
+ self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
+ crawler.settings
+ )
+ downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
self.downloader: Downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
- self._spider_closed_callback = spider_closed_callback
+ self._spider_closed_callback: Callable = spider_closed_callback
+ self.start_time: Optional[float] = None
- def _get_scheduler_class(self, settings: BaseSettings) -> type:
+ def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
from scrapy.core.scheduler import BaseScheduler
- scheduler_cls = load_object(settings["SCHEDULER"])
+ scheduler_cls: Type = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
f"The provided scheduler class ({settings['SCHEDULER']})"
@@ -115,7 +127,7 @@ class ExecutionEngine:
"""Gracefully stop the execution engine"""
@inlineCallbacks
- def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
+ def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
@@ -141,7 +153,8 @@ class ExecutionEngine:
return self.close_spider(
self.spider, reason="shutdown"
) # will also close downloader
- return succeed(self.downloader.close())
+ self.downloader.close()
+ return succeed(None)
def pause(self) -> None:
self.paused = True
@@ -209,7 +222,7 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
)
- d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
+ d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type]
d.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
@@ -339,6 +352,7 @@ class ExecutionEngine:
if isinstance(result, Response):
if result.request is None:
result.request = request
+ assert spider is not None
logkws = self.logformatter.crawled(result.request, result, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
@@ -350,10 +364,12 @@ class ExecutionEngine:
)
return result
- def _on_complete(_):
+ def _on_complete(_: Any) -> Any:
+ assert self.slot is not None
self.slot.nextcall.schedule()
return _
+ assert spider is not None
dwld = self.downloader.fetch(request, spider)
dwld.addCallbacks(_on_success)
dwld.addBoth(_on_complete)
@@ -362,7 +378,7 @@ class ExecutionEngine:
@inlineCallbacks
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
- ):
+ ) -> Generator[Deferred, Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
@@ -471,7 +487,7 @@ class ExecutionEngine:
return dfd
@property
- def open_spiders(self) -> list:
+ def open_spiders(self) -> List[Spider]:
warnings.warn(
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
category=ScrapyDeprecationWarning,
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 3c46e3a5f..3fb0bbaff 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -2,13 +2,15 @@ import json
import logging
from abc import abstractmethod
from pathlib import Path
-from typing import Optional, Type, TypeVar
+from typing import Any, Optional, Type, TypeVar, cast
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
+from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
@@ -20,10 +22,10 @@ class BaseSchedulerMeta(type):
Metaclass to check scheduler classes against the necessary interface
"""
- def __instancecheck__(cls, instance):
+ def __instancecheck__(cls, instance: Any) -> bool:
return cls.__subclasscheck__(type(instance))
- def __subclasscheck__(cls, subclass):
+ def __subclasscheck__(cls, subclass: type) -> bool:
return (
hasattr(subclass, "has_pending_requests")
and callable(subclass.has_pending_requests)
@@ -168,26 +170,26 @@ class Scheduler(BaseScheduler):
def __init__(
self,
- dupefilter,
+ dupefilter: BaseDupeFilter,
jobdir: Optional[str] = None,
dqclass=None,
mqclass=None,
logunser: bool = False,
- stats=None,
+ stats: Optional[StatsCollector] = None,
pqclass=None,
crawler: Optional[Crawler] = None,
):
- self.df = dupefilter
- self.dqdir = self._dqdir(jobdir)
+ self.df: BaseDupeFilter = dupefilter
+ self.dqdir: Optional[str] = self._dqdir(jobdir)
self.pqclass = pqclass
self.dqclass = dqclass
self.mqclass = mqclass
- self.logunser = logunser
- self.stats = stats
- self.crawler = crawler
+ self.logunser: bool = logunser
+ self.stats: Optional[StatsCollector] = stats
+ self.crawler: Optional[Crawler] = crawler
@classmethod
- def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
+ def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
@@ -242,6 +244,7 @@ class Scheduler(BaseScheduler):
self.df.log(request, self.spider)
return False
dqok = self._dqpush(request)
+ assert self.stats is not None
if dqok:
self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
else:
@@ -259,7 +262,8 @@ class Scheduler(BaseScheduler):
Increment the appropriate stats, such as: ``scheduler/dequeued``,
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
"""
- request = self.mqs.pop()
+ request: Optional[Request] = self.mqs.pop()
+ assert self.stats is not None
if request is not None:
self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
else:
@@ -295,6 +299,7 @@ class Scheduler(BaseScheduler):
extra={"spider": self.spider},
)
self.logunser = False
+ assert self.stats is not None
self.stats.inc_value("scheduler/unserializable", spider=self.spider)
return False
else:
@@ -351,7 +356,7 @@ class Scheduler(BaseScheduler):
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
- return json.load(f)
+ return cast(list, json.load(f))
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8468b8419..a85f6a661 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -15,6 +15,7 @@ from typing import (
Optional,
Set,
Tuple,
+ Type,
Union,
)
@@ -26,6 +27,9 @@ from scrapy import Spider, signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
+from scrapy.logformatter import LogFormatter
+from scrapy.pipelines import ItemPipelineManager
+from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import (
aiter_errback,
defer_fail,
@@ -96,16 +100,20 @@ class Slot:
class Scraper:
def __init__(self, crawler: Crawler) -> None:
self.slot: Optional[Slot] = None
- self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
- itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])
- self.itemproc = itemproc_cls.from_crawler(crawler)
- self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")
- self.crawler = crawler
- self.signals = crawler.signals
- self.logformatter = crawler.logformatter
+ self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
+ crawler
+ )
+ itemproc_cls: Type[ItemPipelineManager] = load_object(
+ crawler.settings["ITEM_PROCESSOR"]
+ )
+ self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
+ self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
+ self.crawler: Crawler = crawler
+ self.signals: SignalManager = crawler.signals
+ self.logformatter: LogFormatter = crawler.logformatter
@inlineCallbacks
- def open_spider(self, spider: Spider):
+ def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
"""Open the given spider for scraping and allocate resources for it"""
self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
yield self.itemproc.open_spider(spider)
@@ -135,7 +143,8 @@ class Scraper:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
- def finish_scraping(_):
+ def finish_scraping(_: Any) -> Any:
+ assert self.slot is not None
self.slot.finish_response(result, request)
self._check_if_closing(spider)
self._scrape_next(spider)
@@ -205,9 +214,9 @@ class Scraper:
else: # result is a Failure
# TODO: properly type adding this attribute to a Failure
result.request = request # type: ignore[attr-defined]
- warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
if request.errback:
+ warn_on_generator_with_return_value(spider, request.errback)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
@@ -338,7 +347,7 @@ class Scraper:
def _itemproc_finished(
self, output: Any, item: Any, response: Response, spider: Spider
- ) -> None:
+ ) -> Deferred:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing
self.slot.itemproc_size -= 1
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c21985b18..dcf1a6dbc 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -13,6 +13,8 @@ from typing import (
Callable,
Generator,
Iterable,
+ List,
+ Optional,
Tuple,
Union,
cast,
@@ -25,6 +27,7 @@ from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
+from scrapy.settings import BaseSettings
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
@@ -41,22 +44,22 @@ logger = logging.getLogger(__name__)
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
-def _isiterable(o) -> bool:
+def _isiterable(o: Any) -> bool:
return isinstance(o, (Iterable, AsyncIterable))
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
- def __init__(self, *middlewares):
+ def __init__(self, *middlewares: Any):
super().__init__(*middlewares)
self.downgrade_warning_done = False
@classmethod
- def _get_mwlist_from_settings(cls, settings):
+ def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]:
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
- def _add_middleware(self, mw):
+ def _add_middleware(self, mw: Any) -> None:
super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
@@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
- def process_sync(iterable: Iterable):
+ def process_sync(iterable: Iterable) -> Generator:
try:
for r in iterable:
yield r
@@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise
recover_to.extend(exception_result)
- async def process_async(iterable: AsyncIterable):
+ async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
try:
async for r in iterable:
yield r
@@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
- return MutableChain(result, recovered) # type: ignore[arg-type]
+ return MutableChain(result, recovered)
def scrape_response(
self,
@@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
return dfd
- def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
+ def process_start_requests(
+ self, start_requests: Iterable[Request], spider: Spider
+ ) -> Deferred:
return self._process_chain("process_start_requests", start_requests, spider)
# This method is only needed until _async compatibility methods are removed.
@@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _get_async_method_pair(
mw: Any, methodname: str
) -> Union[None, Callable, Tuple[Callable, Callable]]:
- normal_method = getattr(mw, methodname, None)
+ normal_method: Optional[Callable] = getattr(mw, methodname, None)
methodname_async = methodname + "_async"
- async_method = getattr(mw, methodname_async, None)
+ async_method: Optional[Callable] = getattr(mw, methodname_async, None)
if not async_method:
return normal_method
if not normal_method:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 397817d6f..256f6e2c5 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,7 @@ import logging
import pprint
import signal
import warnings
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Type, Union
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
+from scrapy.logformatter import LogFormatter
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
+from scrapy.statscollectors import StatsCollector
from scrapy.utils.log import (
LogCounterHandler,
configure_logging,
@@ -49,20 +51,25 @@ logger = logging.getLogger(__name__)
class Crawler:
- def __init__(self, spidercls, settings=None, init_reactor: bool = False):
+ def __init__(
+ self,
+ spidercls: Type[Spider],
+ settings: Union[None, dict, Settings] = None,
+ init_reactor: bool = False,
+ ):
if isinstance(spidercls, Spider):
raise ValueError("The spidercls argument must be a class, not an object")
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
- self.spidercls = spidercls
- self.settings = settings.copy()
+ self.spidercls: Type[Spider] = spidercls
+ self.settings: Settings = settings.copy()
self.spidercls.update_settings(self.settings)
- self.signals = SignalManager(self)
+ self.signals: SignalManager = SignalManager(self)
- self.stats = load_object(self.settings["STATS_CLASS"])(self)
+ self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
@@ -80,8 +87,8 @@ class Crawler:
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
- lf_cls = load_object(self.settings["LOG_FORMATTER"])
- self.logformatter = lf_cls.from_crawler(self)
+ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
+ self.logformatter: LogFormatter = lf_cls.from_crawler(self)
self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@@ -89,8 +96,8 @@ class Crawler:
crawler=self,
)
- reactor_class = self.settings["TWISTED_REACTOR"]
- event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
+ reactor_class: str = self.settings["TWISTED_REACTOR"]
+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
@@ -104,11 +111,11 @@ class Crawler:
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
- self.extensions = ExtensionManager.from_crawler(self)
+ self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
self.settings.freeze()
- self.crawling = False
- self.spider = None
+ self.crawling: bool = False
+ self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 7f202b8b8..fedd02805 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -39,7 +39,7 @@ class DontCloseSpider(Exception):
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
- def __init__(self, reason="cancelled"):
+ def __init__(self, reason: str = "cancelled"):
super().__init__()
self.reason = reason
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 560006c95..7cb379b46 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -1,8 +1,11 @@
import logging
import os
+from typing import Any, Dict, Optional, Union
from twisted.python.failure import Failure
+from scrapy import Request, Spider
+from scrapy.http import Response
from scrapy.utils.request import referer_str
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
@@ -52,7 +55,7 @@ class LogFormatter:
}
"""
- def crawled(self, request, response, spider):
+ def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
"""Logs a message when the crawler finds a webpage."""
request_flags = f" {str(request.flags)}" if request.flags else ""
response_flags = f" {str(response.flags)}" if response.flags else ""
@@ -70,8 +73,11 @@ class LogFormatter:
},
}
- def scraped(self, item, response, spider):
+ def scraped(
+ self, item: Any, response: Union[Response, Failure], spider: Spider
+ ) -> dict:
"""Logs a message when an item is scraped by a spider."""
+ src: Any
if isinstance(response, Failure):
src = response.getErrorMessage()
else:
@@ -85,7 +91,9 @@ class LogFormatter:
},
}
- def dropped(self, item, exception, response, spider):
+ def dropped(
+ self, item: Any, exception: BaseException, response: Response, spider: Spider
+ ) -> dict:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
"level": logging.WARNING,
@@ -96,7 +104,9 @@ class LogFormatter:
},
}
- def item_error(self, item, exception, response, spider):
+ def item_error(
+ self, item: Any, exception, response: Response, spider: Spider
+ ) -> dict:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.
@@ -110,7 +120,9 @@ class LogFormatter:
},
}
- def spider_error(self, failure, request, response, spider):
+ def spider_error(
+ self, failure: Failure, request: Request, response: Response, spider: Spider
+ ) -> dict:
"""Logs an error message from a spider.
.. versionadded:: 2.0
@@ -124,13 +136,19 @@ class LogFormatter:
},
}
- def download_error(self, failure, request, spider, errmsg=None):
+ def download_error(
+ self,
+ failure: Failure,
+ request: Request,
+ spider: Spider,
+ errmsg: Optional[str] = None,
+ ) -> dict:
"""Logs a download error message from a spider (typically coming from
the engine).
.. versionadded:: 2.0
"""
- args = {"request": request}
+ args: Dict[str, Any] = {"request": request}
if errmsg:
msg = DOWNLOADERRORMSG_LONG
args["errmsg"] = errmsg
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f82d722fa..03e92b565 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
import logging
import pprint
from collections import defaultdict, deque
-from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
+from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
from twisted.internet.defer import Deferred
@@ -30,7 +30,7 @@ class MiddlewareManager:
self._add_middleware(mw)
@classmethod
- def _get_mwlist_from_settings(cls, settings: Settings) -> list:
+ def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
raise NotImplementedError
@classmethod
@@ -67,17 +67,17 @@ class MiddlewareManager:
def from_crawler(cls, crawler):
return cls.from_settings(crawler.settings, crawler)
- def _add_middleware(self, mw) -> None:
+ def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
- def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
+ def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
return process_parallel(methods, obj, *args)
- def _process_chain(self, methodname: str, obj, *args) -> Deferred:
+ def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
return process_chain(methods, obj, *args)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index df574a0a1..c97d71fb6 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -3,7 +3,11 @@ Item pipeline
See documentation in docs/item-pipeline.rst
"""
+from typing import Any, List
+from twisted.internet.defer import Deferred
+
+from scrapy import Spider
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_f_from_coro_f
@@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"
@classmethod
- def _get_mwlist_from_settings(cls, settings):
+ def _get_mwlist_from_settings(cls, settings) -> List[Any]:
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
- def _add_middleware(self, pipe):
+ def _add_middleware(self, pipe: Any) -> None:
super()._add_middleware(pipe)
if hasattr(pipe, "process_item"):
self.methods["process_item"].append(
deferred_f_from_coro_f(pipe.process_item)
)
- def process_item(self, item, spider):
+ def process_item(self, item: Any, spider: Spider) -> Deferred:
return self._process_chain("process_item", item, spider)
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index d7e3bce91..f6df191d8 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,13 +1,16 @@
+from typing import Any, List, Tuple
+
from pydispatch import dispatcher
+from twisted.internet.defer import Deferred
from scrapy.utils import signal as _signal
class SignalManager:
- def __init__(self, sender=dispatcher.Anonymous):
- self.sender = sender
+ def __init__(self, sender: Any = dispatcher.Anonymous):
+ self.sender: Any = sender
- def connect(self, receiver, signal, **kwargs):
+ def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
"""
Connect a receiver function to a signal.
@@ -22,18 +25,18 @@ class SignalManager:
:type signal: object
"""
kwargs.setdefault("sender", self.sender)
- return dispatcher.connect(receiver, signal, **kwargs)
+ dispatcher.connect(receiver, signal, **kwargs)
- def disconnect(self, receiver, signal, **kwargs):
+ def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
"""
Disconnect a receiver function from a signal. This has the
opposite effect of the :meth:`connect` method, and the arguments
are the same.
"""
kwargs.setdefault("sender", self.sender)
- return dispatcher.disconnect(receiver, signal, **kwargs)
+ dispatcher.disconnect(receiver, signal, **kwargs)
- def send_catch_log(self, signal, **kwargs):
+ def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]:
"""
Send a signal, catch exceptions and log them.
@@ -43,7 +46,7 @@ class SignalManager:
kwargs.setdefault("sender", self.sender)
return _signal.send_catch_log(signal, **kwargs)
- def send_catch_log_deferred(self, signal, **kwargs):
+ def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
"""
Like :meth:`send_catch_log` but supports returning
:class:`~twisted.internet.defer.Deferred` objects from signal handlers.
@@ -57,7 +60,7 @@ class SignalManager:
kwargs.setdefault("sender", self.sender)
return _signal.send_catch_log_deferred(signal, **kwargs)
- def disconnect_all(self, signal, **kwargs):
+ def disconnect_all(self, signal: Any, **kwargs: Any) -> None:
"""
Disconnect all receivers from the given signal.
@@ -65,4 +68,4 @@ class SignalManager:
:type signal: object
"""
kwargs.setdefault("sender", self.sender)
- return _signal.disconnect_all(signal, **kwargs)
+ _signal.disconnect_all(signal, **kwargs)
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index dd3c32737..15193aac5 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats
"""
import logging
import pprint
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from scrapy import Spider
+
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
+StatsT = Dict[str, Any]
+
+
class StatsCollector:
- def __init__(self, crawler):
- self._dump = crawler.settings.getbool("STATS_DUMP")
- self._stats = {}
+ def __init__(self, crawler: "Crawler"):
+ self._dump: bool = crawler.settings.getbool("STATS_DUMP")
+ self._stats: StatsT = {}
- def get_value(self, key, default=None, spider=None):
+ def get_value(
+ self, key: str, default: Any = None, spider: Optional[Spider] = None
+ ) -> Any:
return self._stats.get(key, default)
- def get_stats(self, spider=None):
+ def get_stats(self, spider: Optional[Spider] = None) -> StatsT:
return self._stats
- def set_value(self, key, value, spider=None):
+ def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = value
- def set_stats(self, stats, spider=None):
+ def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
self._stats = stats
- def inc_value(self, key, count=1, start=0, spider=None):
+ def inc_value(
+ self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
+ ) -> None:
d = self._stats
d[key] = d.setdefault(key, start) + count
- def max_value(self, key, value, spider=None):
+ def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = max(self._stats.setdefault(key, value), value)
- def min_value(self, key, value, spider=None):
+ def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = min(self._stats.setdefault(key, value), value)
- def clear_stats(self, spider=None):
+ def clear_stats(self, spider: Optional[Spider] = None) -> None:
self._stats.clear()
- def open_spider(self, spider):
+ def open_spider(self, spider: Spider) -> None:
pass
- def close_spider(self, spider, reason):
+ def close_spider(self, spider: Spider, reason: str) -> None:
if self._dump:
logger.info(
"Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
@@ -48,34 +61,38 @@ class StatsCollector:
)
self._persist_stats(self._stats, spider)
- def _persist_stats(self, stats, spider):
+ def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
pass
class MemoryStatsCollector(StatsCollector):
- def __init__(self, crawler):
+ def __init__(self, crawler: "Crawler"):
super().__init__(crawler)
- self.spider_stats = {}
+ self.spider_stats: Dict[str, StatsT] = {}
- def _persist_stats(self, stats, spider):
+ def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
self.spider_stats[spider.name] = stats
class DummyStatsCollector(StatsCollector):
- def get_value(self, key, default=None, spider=None):
+ def get_value(
+ self, key: str, default: Any = None, spider: Optional[Spider] = None
+ ) -> Any:
return default
- def set_value(self, key, value, spider=None):
+ def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass
- def set_stats(self, stats, spider=None):
+ def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
pass
- def inc_value(self, key, count=1, start=0, spider=None):
+ def inc_value(
+ self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
+ ) -> None:
pass
- def max_value(self, key, value, spider=None):
+ def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass
- def min_value(self, key, value, spider=None):
+ def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6ae27dc29..2ce4725f4 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -2,6 +2,7 @@ import logging
import sys
import warnings
from logging.config import dictConfig
+from typing import Tuple
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
@@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versions
logger = logging.getLogger(__name__)
-def failure_to_exc_info(failure):
+def failure_to_exc_info(failure: Failure):
"""Extract exc_info from Failure instances"""
if isinstance(failure, Failure):
return (failure.type, failure.value, failure.getTracebackObject())
@@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler):
self.crawler.stats.inc_value(sname)
-def logformatter_adapter(logkws):
+def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
"""
Helper that takes the dictionary output from the methods in LogFormatter
and adapts it into a tuple of positional arguments for logger.log calls,
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f9f9c0d5b..d861c9ab6 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -10,6 +10,7 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
+from typing import TYPE_CHECKING, Any, Callable, Union
from w3lib.html import replace_entities
@@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.python import flatten, to_unicode
+if TYPE_CHECKING:
+ from scrapy import Spider
+
+
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
@@ -34,7 +39,7 @@ def arg_to_iter(arg):
return [arg]
-def load_object(path):
+def load_object(path: Union[str, Callable]) -> Any:
"""Load an object given its absolute object path, and return it.
The object can be the import path of a class, function, variable or an
@@ -249,7 +254,7 @@ def is_generator_with_return_value(callable):
return _generator_callbacks_cache[callable]
-def warn_on_generator_with_return_value(spider, callable):
+def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
"""
Logs a warning if a callable is a generator function and includes
a 'return' statement with a value different than None
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 7f67d036a..f1b9239e6 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,9 +1,11 @@
import asyncio
import sys
from contextlib import suppress
+from typing import Any, Callable, Dict, Optional, Sequence
from warnings import catch_warnings, filterwarnings, warn
from twisted.internet import asyncioreactor, error
+from twisted.internet.base import DelayedCall
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
@@ -34,23 +36,23 @@ class CallLaterOnce:
it hasn't been already scheduled since the last time it ran.
"""
- def __init__(self, func, *a, **kw):
- self._func = func
- self._a = a
- self._kw = kw
- self._call = None
+ def __init__(self, func: Callable, *a: Any, **kw: Any):
+ self._func: Callable = func
+ self._a: Sequence[Any] = a
+ self._kw: Dict[str, Any] = kw
+ self._call: Optional[DelayedCall] = None
- def schedule(self, delay=0):
+ def schedule(self, delay: float = 0) -> None:
from twisted.internet import reactor
if self._call is None:
self._call = reactor.callLater(delay, self)
- def cancel(self):
+ def cancel(self) -> None:
if self._call:
self._call.cancel()
- def __call__(self):
+ def __call__(self) -> Any:
self._call = None
return self._func(*self._a, **self._kw)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index b95786d35..9e7ddd827 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,6 +1,8 @@
"""Helper functions for working with signals"""
import collections.abc
import logging
+from typing import Any as TypingAny
+from typing import List, Tuple
from pydispatch.dispatcher import (
Anonymous,
@@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
-def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
+def send_catch_log(
+ signal=Any, sender=Anonymous, *arguments, **named
+) -> List[Tuple[TypingAny, TypingAny]]:
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
@@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
)
dont_log += (StopDownload,)
spider = named.get("spider", None)
- responses = []
+ responses: List[Tuple[TypingAny, TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
+ result: TypingAny
try:
response = robustApply(
receiver, signal=signal, sender=sender, *arguments, **named
diff --git a/tox.ini b/tox.ini
index 5a9d9cf29..d96a278ea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,13 +37,13 @@ install_command =
[testenv:typing]
basepython = python3
deps =
- mypy==1.0.1
+ mypy==1.2.0
types-attrs==19.1.0
- types-lxml==2023.2.11
- types-Pillow==9.4.0.16
- types-Pygments==2.14.0.5
- types-pyOpenSSL==23.0.0.4
- types-setuptools==67.4.0.1
+ types-lxml==2023.3.28
+ types-Pillow==9.4.0.19
+ types-Pygments==2.14.0.7
+ types-pyOpenSSL==23.1.0.1
+ types-setuptools==67.6.0.7
commands =
mypy --show-error-codes {posargs: scrapy tests}