Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/extensions, part 1. (#6276)
Andrey Rakhmatullin421e08dd4a4d
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex 4307b4170..812b3553c 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -4,20 +4,31 @@ conditions are met.See documentation in docs/topics/extensions.rst"""+from __future__ import annotations+import loggingfrom collections import defaultdict+from typing import TYPE_CHECKING, Any, DefaultDict, Dict++from twisted.python.failure import Failure-from scrapy import signals+from scrapy import Request, Spider, signals+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.http import Response++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selflogger = logging.getLogger(__name__)class CloseSpider:- def __init__(self, crawler):- self.crawler = crawler+ def __init__(self, crawler: Crawler):+ self.crawler: Crawler = crawler- self.close_on = {+ self.close_on: Dict[str, Any] = {"timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),@@ -28,7 +39,7 @@ class CloseSpider:if not any(self.close_on.values()):raise NotConfigured- self.counter = defaultdict(int)+ self.counter: DefaultDict[str, int] = defaultdict(int)if self.close_on.get("errorcount"):crawler.signals.connect(self.error_count, signal=signals.spider_error)@@ -39,8 +50,8 @@ class CloseSpider:if self.close_on.get("itemcount"):crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)if self.close_on.get("timeout_no_item"):- self.timeout_no_item = self.close_on["timeout_no_item"]- self.items_in_period = 0+ self.timeout_no_item: int = self.close_on["timeout_no_item"]+ self.items_in_period: int = 0crawler.signals.connect(self.spider_opened_no_item, signal=signals.spider_opened)@@ -50,22 +61,25 @@ class CloseSpider:crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- def error_count(self, failure, response, spider):+ def error_count(self, failure: Failure, response: Response, spider: Spider) -> None:self.counter["errorcount"] += 1if self.counter["errorcount"] == self.close_on["errorcount"]:+ assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_errorcount")- def page_count(self, response, request, spider):+ def page_count(self, response: Response, request: Request, spider: Spider) -> None:self.counter["pagecount"] += 1if self.counter["pagecount"] == self.close_on["pagecount"]:+ assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_pagecount")- def spider_opened(self, spider):+ def spider_opened(self, spider: Spider) -> None:from twisted.internet import reactor+ assert self.crawler.engineself.task = reactor.callLater(self.close_on["timeout"],self.crawler.engine.close_spider,@@ -73,21 +87,22 @@ class CloseSpider:reason="closespider_timeout",)- def item_scraped(self, item, spider):+ def item_scraped(self, item: Any, spider: Spider) -> None:self.counter["itemcount"] += 1if self.counter["itemcount"] == self.close_on["itemcount"]:+ assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_itemcount")- def spider_closed(self, spider):- task = getattr(self, "task", False)+ def spider_closed(self, spider: Spider) -> None:+ task = getattr(self, "task", None)if task and task.active():task.cancel()- task_no_item = getattr(self, "task_no_item", False)+ task_no_item = getattr(self, "task_no_item", None)if task_no_item and task_no_item.running:task_no_item.stop()- def spider_opened_no_item(self, spider):+ def spider_opened_no_item(self, spider: Spider) -> None:from twisted.internet import taskself.task_no_item = task.LoopingCall(self._count_items_produced, spider)@@ -98,10 +113,10 @@ class CloseSpider:f"{self.timeout_no_item} seconds.")- def item_scraped_no_item(self, item, spider):+ def item_scraped_no_item(self, item: Any, spider: Spider) -> None:self.items_in_period += 1- def _count_items_produced(self, spider):+ def _count_items_produced(self, spider: Spider) -> None:if self.items_in_period >= 1:self.items_in_period = 0else:@@ -109,4 +124,5 @@ class CloseSpider:f"Closing spider since no items were produced in the last "f"{self.timeout_no_item} seconds.")+ assert self.crawler.engineself.crawler.engine.close_spider(spider, "closespider_timeout_no_item")diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex 717c249d9..f3ac19623 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -2,18 +2,28 @@Extension for collecting core stats like items scraped and start/finish times"""+from __future__ import annotations+from datetime import datetime, timezone+from typing import TYPE_CHECKING, Any, Optional++from scrapy import Spider, signals+from scrapy.crawler import Crawler+from scrapy.statscollectors import StatsCollector-from scrapy import signals+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selfclass CoreStats:- def __init__(self, stats):- self.stats = stats- self.start_time = None+ def __init__(self, stats: StatsCollector):+ self.stats: StatsCollector = stats+ self.start_time: Optional[datetime] = None@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:+ assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)@@ -22,11 +32,12 @@ class CoreStats:crawler.signals.connect(o.response_received, signal=signals.response_received)return o- def spider_opened(self, spider):+ def spider_opened(self, spider: Spider) -> None:self.start_time = datetime.now(tz=timezone.utc)self.stats.set_value("start_time", self.start_time, spider=spider)- def spider_closed(self, spider, reason):+ def spider_closed(self, spider: Spider, reason: str) -> None:+ assert self.start_time is not Nonefinish_time = datetime.now(tz=timezone.utc)elapsed_time = finish_time - self.start_timeelapsed_time_seconds = elapsed_time.total_seconds()@@ -36,13 +47,13 @@ class CoreStats:self.stats.set_value("finish_time", finish_time, spider=spider)self.stats.set_value("finish_reason", reason, spider=spider)- def item_scraped(self, item, spider):+ def item_scraped(self, item: Any, spider: Spider) -> None:self.stats.inc_value("item_scraped_count", spider=spider)- def response_received(self, spider):+ def response_received(self, spider: Spider) -> None:self.stats.inc_value("response_received_count", spider=spider)- def item_dropped(self, item, spider, exception):+ def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:reason = exception.__class__.__name__self.stats.inc_value("item_dropped_count", spider=spider)self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.pyindex 1b6c7777f..26726b662 100644--- a/scrapy/extensions/debug.py+++ b/scrapy/extensions/debug.py@@ -4,22 +4,31 @@ Extensions for debugging ScrapySee documentation in docs/topics/extensions.rst"""+from __future__ import annotations+import loggingimport signalimport sysimport threadingimport tracebackfrom pdb import Pdb+from types import FrameType+from typing import TYPE_CHECKING, Optional+from scrapy.crawler import Crawlerfrom scrapy.utils.engine import format_engine_statusfrom scrapy.utils.trackref import format_live_refs+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+logger = logging.getLogger(__name__)class StackTraceDump:- def __init__(self, crawler=None):- self.crawler = crawler+ def __init__(self, crawler: Crawler):+ self.crawler: Crawler = crawlertry:signal.signal(signal.SIGUSR2, self.dump_stacktrace)signal.signal(signal.SIGQUIT, self.dump_stacktrace)@@ -28,10 +37,11 @@ class StackTraceDump:pass@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- def dump_stacktrace(self, signum, frame):+ def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None:+ assert self.crawler.enginelog_args = {"stackdumps": self._thread_stacks(),"enginestatus": format_engine_status(self.crawler.engine),@@ -44,7 +54,7 @@ class StackTraceDump:extra={"crawler": self.crawler},)- def _thread_stacks(self):+ def _thread_stacks(self) -> str:id2name = dict((th.ident, th.name) for th in threading.enumerate())dumps = ""for id_, frame in sys._current_frames().items():@@ -55,12 +65,13 @@ class StackTraceDump:class Debugger:- def __init__(self):+ def __init__(self) -> None:try:signal.signal(signal.SIGUSR2, self._enter_debugger)except AttributeError:# win32 platforms don't support SIGUSR signalspass- def _enter_debugger(self, signum, frame):+ def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:+ assert framePdb().set_trace(frame.f_back)diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.pyindex 9f63e9c4b..2388afa75 100644--- a/scrapy/extensions/logstats.py+++ b/scrapy/extensions/logstats.py@@ -1,9 +1,18 @@+from __future__ import annotations+import logging+from typing import TYPE_CHECKING, Optional, Tuple, Unionfrom twisted.internet import task-from scrapy import signals+from scrapy import Spider, signals+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.statscollectors import StatsCollector++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selflogger = logging.getLogger(__name__)@@ -14,30 +23,31 @@ class LogStats:* IPM - Items per Minute"""- def __init__(self, stats, interval=60.0):- self.stats = stats- self.interval = interval- self.multiplier = 60.0 / self.interval- self.task = None+ def __init__(self, stats: StatsCollector, interval: float = 60.0):+ self.stats: StatsCollector = stats+ self.interval: float = interval+ self.multiplier: float = 60.0 / self.interval+ self.task: Optional[task.LoopingCall] = None@classmethod- def from_crawler(cls, crawler):- interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")+ def from_crawler(cls, crawler: Crawler) -> Self:+ interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")if not interval:raise NotConfigured+ assert crawler.statso = cls(crawler.stats, interval)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)return o- def spider_opened(self, spider):- self.pagesprev = 0- self.itemsprev = 0+ def spider_opened(self, spider: Spider) -> None:+ self.pagesprev: int = 0+ self.itemsprev: int = 0self.task = task.LoopingCall(self.log, spider)self.task.start(self.interval)- def log(self, spider):+ def log(self, spider: Spider) -> None:self.calculate_stats()msg = (@@ -52,14 +62,14 @@ class LogStats:}logger.info(msg, log_args, extra={"spider": spider})- def calculate_stats(self):- self.items = self.stats.get_value("item_scraped_count", 0)- self.pages = self.stats.get_value("response_received_count", 0)- self.irate = (self.items - self.itemsprev) * self.multiplier- self.prate = (self.pages - self.pagesprev) * self.multiplier+ def calculate_stats(self) -> None:+ self.items: int = self.stats.get_value("item_scraped_count", 0)+ self.pages: int = self.stats.get_value("response_received_count", 0)+ self.irate: float = (self.items - self.itemsprev) * self.multiplier+ self.prate: float = (self.pages - self.pagesprev) * self.multiplierself.pagesprev, self.itemsprev = self.pages, self.items- def spider_closed(self, spider, reason):+ def spider_closed(self, spider: Spider, reason: str) -> None:if self.task and self.task.running:self.task.stop()@@ -67,7 +77,9 @@ class LogStats:self.stats.set_value("responses_per_minute", rpm_final)self.stats.set_value("items_per_minute", ipm_final)- def calculate_final_stats(self, spider):+ def calculate_final_stats(+ self, spider: Spider+ ) -> Union[Tuple[None, None], Tuple[float, float]]:start_time = self.stats.get_value("start_time")finished_time = self.stats.get_value("finished_time")diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.pyindex 03ede0681..f304e1bf2 100644--- a/scrapy/extensions/memdebug.py+++ b/scrapy/extensions/memdebug.py@@ -4,26 +4,36 @@ MemoryDebugger extensionSee documentation in docs/topics/extensions.rst"""+from __future__ import annotations+import gc+from typing import TYPE_CHECKING-from scrapy import signals+from scrapy import Spider, signals+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.trackref import live_refs+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+class MemoryDebugger:- def __init__(self, stats):- self.stats = stats+ def __init__(self, stats: StatsCollector):+ self.stats: StatsCollector = stats@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("MEMDEBUG_ENABLED"):raise NotConfigured+ assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)return o- def spider_closed(self, spider, reason):+ def spider_closed(self, spider: Spider, reason: str) -> None:gc.collect()self.stats.set_value("memdebug/gc_garbage_count", len(gc.garbage), spider=spiderdiff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex 4d4501c44..9de06b24d 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -4,24 +4,32 @@ MemoryUsage extensionSee documentation in docs/topics/extensions.rst"""+from __future__ import annotations+import loggingimport socketimport sysfrom importlib import import_modulefrom pprint import pformat+from typing import TYPE_CHECKING, Listfrom twisted.internet import taskfrom scrapy import signals+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.mail import MailSenderfrom scrapy.utils.engine import get_engine_status+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+logger = logging.getLogger(__name__)class MemoryUsage:- def __init__(self, crawler):+ def __init__(self, crawler: Crawler):if not crawler.settings.getbool("MEMUSAGE_ENABLED"):raise NotConfiguredtry:@@ -30,32 +38,33 @@ class MemoryUsage:except ImportError:raise NotConfigured- self.crawler = crawler- self.warned = False- self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")- self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024- self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024- self.check_interval = crawler.settings.getfloat(+ self.crawler: Crawler = crawler+ self.warned: bool = False+ self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")+ self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024+ self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024+ self.check_interval: float = crawler.settings.getfloat("MEMUSAGE_CHECK_INTERVAL_SECONDS")- self.mail = MailSender.from_settings(crawler.settings)+ self.mail: MailSender = MailSender.from_settings(crawler.settings)crawler.signals.connect(self.engine_started, signal=signals.engine_started)crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- def get_virtual_size(self):- size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss+ def get_virtual_size(self) -> int:+ size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrssif sys.platform != "darwin":# on macOS ru_maxrss is in bytes, on Linux it is in KBsize *= 1024return size- def engine_started(self):+ def engine_started(self) -> None:+ assert self.crawler.statsself.crawler.stats.set_value("memusage/startup", self.get_virtual_size())- self.tasks = []+ self.tasks: List[task.LoopingCall] = []tsk = task.LoopingCall(self.update)self.tasks.append(tsk)tsk.start(self.check_interval, now=True)@@ -68,15 +77,18 @@ class MemoryUsage:self.tasks.append(tsk)tsk.start(self.check_interval, now=True)- def engine_stopped(self):+ def engine_stopped(self) -> None:for tsk in self.tasks:if tsk.running:tsk.stop()- def update(self):+ def update(self) -> None:+ assert self.crawler.statsself.crawler.stats.max_value("memusage/max", self.get_virtual_size())- def _check_limit(self):+ def _check_limit(self) -> None:+ assert self.crawler.engine+ assert self.crawler.statspeak_mem_usage = self.get_virtual_size()if peak_mem_usage > self.limit:self.crawler.stats.set_value("memusage/limit_reached", 1)@@ -106,9 +118,10 @@ class MemoryUsage:{"virtualsize": peak_mem_usage / 1024 / 1024},)- def _check_warning(self):+ def _check_warning(self) -> None:if self.warned: # warn only oncereturn+ assert self.crawler.statsif self.get_virtual_size() > self.warning:self.crawler.stats.set_value("memusage/warning_reached", 1)mem = self.warning / 1024 / 1024@@ -126,8 +139,10 @@ class MemoryUsage:self.crawler.stats.set_value("memusage/warning_notified", 1)self.warned = True- def _send_report(self, rcpts, subject):+ def _send_report(self, rcpts: List[str], subject: str) -> None:"""send notification mail with some additional useful info"""+ assert self.crawler.engine+ assert self.crawler.statsstats = self.crawler.statss = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"diff --git a/scrapy/mail.py b/scrapy/mail.pyindex 4b18b6003..dce33fcdf 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -4,6 +4,8 @@ Mail sending helpersSee documentation in docs/topics/email.rst"""+from __future__ import annotations+import loggingfrom email import encoders as Encodersfrom email.mime.base import MIMEBase@@ -12,14 +14,20 @@ from email.mime.nonmultipart import MIMENonMultipartfrom email.mime.text import MIMETextfrom email.utils import formatdatefrom io import BytesIO+from typing import TYPE_CHECKINGfrom twisted import version as twisted_versionfrom twisted.internet import defer, sslfrom twisted.python.versions import Version+from scrapy.settings import BaseSettingsfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import to_bytes+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+logger = logging.getLogger(__name__)@@ -56,7 +64,7 @@ class MailSender:self.debug = debug@classmethod- def from_settings(cls, settings):+ def from_settings(cls, settings: BaseSettings) -> Self:return cls(smtphost=settings["MAIL_HOST"],mailfrom=settings["MAIL_FROM"],@@ -203,7 +211,7 @@ class MailSender:to_addrs,msg,d,- **factory_keywords+ **factory_keywords,)factory.noisy = Falsereturn factorydiff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex 0b2722663..fdcf484d4 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -1,14 +1,15 @@"""Some debugging functions for working with the Scrapy engine"""+from __future__ import annotations+# used in global tests codefrom time import time # noqa: F401-from typing import TYPE_CHECKING, Any, List, Tuple+from typing import Any, List, Tuple-if TYPE_CHECKING:- from scrapy.core.engine import ExecutionEngine+from scrapy.core.engine import ExecutionEngine-def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:+def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:"""Return a report of the current engine status"""tests = ["time()-engine.start_time",@@ -37,7 +38,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:return checks-def format_engine_status(engine: "ExecutionEngine") -> str:+def format_engine_status(engine: ExecutionEngine) -> str:checks = get_engine_status(engine)s = "Execution engine status\n\n"for test, result in checks:@@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str:return s-def print_engine_status(engine: "ExecutionEngine") -> None:+def print_engine_status(engine: ExecutionEngine) -> None:print(format_engine_status(engine))