Scrapy, a fast high-level web crawling & scraping framework for Python.
Type late Crawler attributes as always set instead of None (#7882)
Adrian508bd7faec44
diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex 7ff8d3464..598edfeb5 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares:class:`scrapy.Spider` subclass and a:class:`scrapy.settings.Settings` object.+ The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`,+ :attr:`request_fingerprinter` and :attr:`stats` attributes get their value+ when the crawl starts, and raise :exc:`RuntimeError` when read before that.++ .. versionchanged:: VERSION+ Those attributes used to be ``None`` before getting their value.+.. attribute:: request_fingerprinterThe request fingerprint builder of this crawler.diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 51caed57f..b6255ec7e 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand):) -> list[Any]:items, requests, opts, depth, spider, callback = argsif opts.pipelines:- assert self.pcrawler.engineitemproc = self.pcrawler.engine.scraper.itemprocif hasattr(itemproc, "process_item_async"):for item in items:diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex b7a1e0fcd..b63d136a4 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -112,7 +112,6 @@ class ExecutionEngine:self.crawler: Crawler = crawlerself.settings: Settings = crawler.settingsself.signals: SignalManager = crawler.signals- assert crawler.logformatterself.logformatter: LogFormatter = crawler.logformatterself._slot: _Slot | None = Noneself.spider: Spider | None = None@@ -299,7 +298,6 @@ class ExecutionEngine:response=None,spider=self.spider,)- assert self.crawler.statsself.crawler.stats.inc_value("spider_exceptions/count")self.crawler.stats.inc_value(f"spider_exceptions/{type(exception).__name__}"@@ -563,17 +561,17 @@ class ExecutionEngine:if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):await maybe_deferred_to_future(d)await self.scraper.open_spider_async()- assert self.crawler.stats- if argument_is_required(self.crawler.stats.open_spider, "spider"):+ stats = self.crawler.stats+ if argument_is_required(stats.open_spider, "spider"):warnings.warn(- f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"+ f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"f" this is deprecated and the argument will not be passed in future Scrapy versions.",ScrapyDeprecationWarning,stacklevel=2,)- self.crawler.stats.open_spider(spider=self.crawler.spider)+ stats.open_spider(spider=self.crawler.spider)else:- self.crawler.stats.open_spider()+ stats.open_spider()await self.signals.send_catch_log_async(signals.spider_opened, spider=self.crawler.spider)@@ -676,20 +674,18 @@ class ExecutionEngine:extra={"spider": spider},)- assert self.crawler.statstry:- if argument_is_required(self.crawler.stats.close_spider, "spider"):+ stats = self.crawler.stats+ if argument_is_required(stats.close_spider, "spider"):warnings.warn(- f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"+ f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument,"f" this is deprecated and the argument will not be passed in future Scrapy versions.",ScrapyDeprecationWarning,stacklevel=2,)- self.crawler.stats.close_spider(- spider=self.crawler.spider, reason=reason- )+ stats.close_spider(spider=self.crawler.spider, reason=reason)else:- self.crawler.stats.close_spider(reason=reason)+ stats.close_spider(reason=reason)except Exception:logger.error("Stats close failure")diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 58e37ce5e..6426b1751 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -120,7 +120,6 @@ class Scraper:self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")self.crawler: Crawler = crawlerself.signals: SignalManager = crawler.signals- assert crawler.logformatterself.logformatter: LogFormatter = crawler.logformatterdef _check_deprecated_itemproc_method(self, method: str) -> None:@@ -355,7 +354,6 @@ class Scraper:assert self.crawler.spiderexc = _failure.valueif isinstance(exc, CloseSpider):- assert self.crawler.engine is not None # typing_schedule_coro(self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled"))@@ -374,11 +372,9 @@ class Scraper:response=response,spider=self.crawler.spider,)- assert self.crawler.stats- self.crawler.stats.inc_value("spider_exceptions/count")- self.crawler.stats.inc_value(- f"spider_exceptions/{_failure.value.__class__.__name__}"- )+ stats = self.crawler.stats+ stats.inc_value("spider_exceptions/count")+ stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")def handle_spider_output(self,@@ -456,7 +452,6 @@ class Scraper:Items are sent to the item pipelines, requests are scheduled."""if isinstance(output, Request):- assert self.crawler.engine is not None # typingself.crawler.engine.crawl(request=output)returnif output is not None:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex e2f726519..44c4ffdcf 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -8,7 +8,7 @@ import signalimport warningsfrom abc import ABC, abstractmethodfrom functools import partial-from typing import TYPE_CHECKING, Any, TypeVar+from typing import TYPE_CHECKING, Any, Generic, TypeVar, overloadfrom twisted.internet.defer import Deferred, DeferredList, inlineCallbacks@@ -58,7 +58,55 @@ logger = logging.getLogger(__name__)_T = TypeVar("_T")+class _LateAttribute(Generic[_T]):+ """Descriptor for a :class:`Crawler` attribute that only gets a value once+ the crawl starts.++ The value is kept in an attribute of the same name prefixed with an+ underscore, and reading it before it is set raises :exc:`RuntimeError`.+ This way the public attribute can be annotated as always set, and its+ users, both in Scrapy and in third-party code, do not need to narrow its+ type on every use. Code that runs before the crawl starts reads the+ underscore-prefixed attribute instead.+ """++ def __set_name__(self, owner: type[Crawler], name: str) -> None:+ self._name = name+ self._private_name = f"_{name}"++ @overload+ def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ...++ @overload+ def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ...++ def __get__(+ self, instance: Crawler | None, owner: type[Crawler]+ ) -> _LateAttribute[_T] | _T:+ if instance is None:+ return self+ value: _T | None = getattr(instance, self._private_name)+ if value is None:+ raise RuntimeError(+ f"Crawler.{self._name} is not set yet. It is set when the "+ "crawl starts, so it can only be used from then on, e.g. "+ "from the spider_opened signal handler onwards."+ )+ return value++ def __set__(self, instance: Crawler, value: _T) -> None:+ setattr(instance, self._private_name, value)++class Crawler:+ engine: _LateAttribute[ExecutionEngine] = _LateAttribute()+ extensions: _LateAttribute[ExtensionManager] = _LateAttribute()+ logformatter: _LateAttribute[LogFormatter] = _LateAttribute()+ request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = (+ _LateAttribute()+ )+ stats: _LateAttribute[StatsCollector] = _LateAttribute()+def __init__(self,spidercls: type[Spider],@@ -83,12 +131,13 @@ class Crawler:self.crawling: bool = Falseself._started: bool = False- self.extensions: ExtensionManager | None = None- self.stats: StatsCollector | None = None- self.logformatter: LogFormatter | None = None- self.request_fingerprinter: RequestFingerprinterProtocol | None = Noneself.spider: Spider | None = None- self.engine: ExecutionEngine | None = None++ self._engine: ExecutionEngine | None = None+ self._extensions: ExtensionManager | None = None+ self._logformatter: LogFormatter | None = None+ self._request_fingerprinter: RequestFingerprinterProtocol | None = None+ self._stats: StatsCollector | None = Nonedef _update_root_log_handler(self) -> None:if get_scrapy_root_handler() is not None:@@ -225,8 +274,8 @@ class Crawler:yield deferred_from_coro(self.engine.start_async())except Exception:self.crawling = False- if self.engine is not None:- yield deferred_from_coro(self.engine.close_async())+ if self._engine is not None:+ yield deferred_from_coro(self._engine.close_async())raiseasync def crawl_async(self, *args: Any, **kwargs: Any) -> None:@@ -255,8 +304,8 @@ class Crawler:await self.engine.start_async()except Exception:self.crawling = False- if self.engine is not None:- await self.engine.close_async()+ if self._engine is not None:+ await self._engine.close_async()raisedef _create_spider(self, *args: Any, **kwargs: Any) -> Spider:@@ -282,7 +331,6 @@ class Crawler:"""if self.crawling:self.crawling = False- assert self.engineif self.engine.running:await self.engine.stop_async()@@ -313,7 +361,7 @@ class Crawler:This method can only be called after the crawl engine has been created,e.g. at signals :signal:`engine_started` or :signal:`spider_opened`."""- if not self.engine:+ if self._engine is None:raise RuntimeError("Crawler.get_downloader_middleware() can only be called after ""the crawl engine has been created."@@ -331,7 +379,7 @@ class Crawler:created, e.g. at signals :signal:`engine_started` or:signal:`spider_opened`."""- if not self.extensions:+ if self._extensions is None:raise RuntimeError("Crawler.get_extension() can only be called after the ""extension manager has been created."@@ -348,7 +396,7 @@ class Crawler:This method can only be called after the crawl engine has been created,e.g. at signals :signal:`engine_started` or :signal:`spider_opened`."""- if not self.engine:+ if self._engine is None:raise RuntimeError("Crawler.get_item_pipeline() can only be called after the ""crawl engine has been created."@@ -365,7 +413,7 @@ class Crawler:This method can only be called after the crawl engine has been created,e.g. at signals :signal:`engine_started` or :signal:`spider_opened`."""- if not self.engine:+ if self._engine is None:raise RuntimeError("Crawler.get_spider_middleware() can only be called after the ""crawl engine has been created."diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex e7ca0ac0e..3ebd98027 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -55,7 +55,6 @@ class HttpCacheMiddleware:@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- assert crawler.statso = cls(crawler.settings, crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex 0d5c9e4c9..b9a26df3a 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -61,7 +61,6 @@ class OffsiteMiddleware:@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex f910d07c8..dd2897ee4 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -94,7 +94,6 @@ def get_retry_request(retry-related job stats"""settings = spider.crawler.settings- assert spider.crawler.statsstats = spider.crawler.statsretry_times = request.meta.get("retry_times", 0) + 1if max_retry_times is None:diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 81a3a887f..d7aa8738c 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.robotstxt import RobotParser+ from scrapy.statscollectors import StatsCollectorlogger = logging.getLogger(__name__)@@ -41,6 +42,7 @@ class RobotsTxtMiddleware:self._default_useragent: str = crawler.settings["USER_AGENT"]self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]self.crawler: Crawler = crawler+ self._stats: StatsCollector = crawler.statsself._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}self._parserimpl: RobotParser = load_object(crawler.settings.get("ROBOTSTXT_PARSER")@@ -78,8 +80,7 @@ class RobotsTxtMiddleware:{"request": request},extra={"spider": self.crawler.spider},)- assert self.crawler.stats- self.crawler.stats.inc_value("robotstxt/forbidden")+ self._stats.inc_value("robotstxt/forbidden")raise IgnoreRequest("Forbidden by robots.txt")async def robot_parser(self, request: Request) -> RobotParser | None:@@ -95,8 +96,6 @@ class RobotsTxtMiddleware:meta={"dont_obey_robotstxt": True},callback=NO_CALLBACK,)- assert self.crawler.engine- assert self.crawler.statstry:resp = await self.crawler.engine.download_async(robotsreq)await self._parse_robots(resp, netloc, request)@@ -109,7 +108,7 @@ class RobotsTxtMiddleware:extra={"spider": self.crawler.spider},)self._robots_error(e, netloc)- self.crawler.stats.inc_value("robotstxt/request_count")+ self._stats.inc_value("robotstxt/request_count")parser = self._parsers[netloc]if isinstance(parser, Deferred):@@ -119,11 +118,8 @@ class RobotsTxtMiddleware:async def _parse_robots(self, response: Response, netloc: str, request: Request) -> None:- assert self.crawler.stats- self.crawler.stats.inc_value("robotstxt/response_count")- self.crawler.stats.inc_value(- f"robotstxt/response_status_count/{response.status}"- )+ self._stats.inc_value("robotstxt/response_count")+ self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")rp = self._parserimpl.from_crawler(self.crawler, response.body)await self.crawler.signals.send_catch_log_async(signal=signals.robots_parsed,@@ -138,8 +134,7 @@ class RobotsTxtMiddleware:def _robots_error(self, exc: Exception, netloc: str) -> None:if not isinstance(exc, IgnoreRequest):key = f"robotstxt/exception_count/{type(exc)}"- assert self.crawler.stats- self.crawler.stats.inc_value(key)+ self._stats.inc_value(key)rp_dfd = self._parsers[netloc]assert isinstance(rp_dfd, Deferred)self._parsers[netloc] = Nonediff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex bafa931de..07de1c2e7 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -43,7 +43,6 @@ class DownloaderStats:def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("DOWNLOADER_STATS"):raise NotConfigured- assert crawler.statsreturn cls(crawler.stats)@_warn_spider_argdiff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 36fb0f97d..09f0be63b 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter):@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- assert crawler.request_fingerprinterdebug = crawler.settings.getbool("DUPEFILTER_DEBUG")return cls(job_dir(crawler.settings),@@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter):self.logger.debug(msg, {"request": request}, extra={"spider": spider})self.logdupes = False- assert spider.crawler.statsspider.crawler.stats.inc_value("dupefilter/filtered")diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex 9cb792e30..5d40e5f9f 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -102,7 +102,6 @@ class CloseSpider:self._close_spider("closespider_pagecount_no_item")def spider_opened(self, spider: Spider) -> None:- assert self.crawler.engineself.task = call_later(self.close_on["timeout"], self._close_spider, "closespider_timeout")@@ -146,5 +145,4 @@ class CloseSpider:self._close_spider("closespider_timeout_no_item")def _close_spider(self, reason: str) -> None:- assert self.crawler.engine_schedule_coro(self.crawler.engine.close_spider_async(reason=reason))diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex 6a5e55992..b464942af 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -26,7 +26,6 @@ class CoreStats:@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.pyindex 5def7509e..8802a3b58 100644--- a/scrapy/extensions/debug.py+++ b/scrapy/extensions/debug.py@@ -45,7 +45,6 @@ class StackTraceDump:return cls(crawler)def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None:- assert self.crawler.enginelog_args = {"stackdumps": self._thread_stacks(),"enginestatus": format_engine_status(self.crawler.engine),diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex e4294ac90..32abc82d3 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -611,7 +611,6 @@ class FeedExporter:logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"slot_type = type(slot.storage).__name__- assert self.crawler.statstry:await ensure_awaitable(slot.storage.store(self._get_file(slot)))except Exception:diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex dbb79b02d..d008d0f67 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -261,7 +261,6 @@ class DbmCacheStorage:extra={"spider": spider},)- assert spider.crawler.request_fingerprinterself._fingerprinter: RequestFingerprinterProtocol = (spider.crawler.request_fingerprinter)@@ -326,7 +325,6 @@ class FilesystemCacheStorage:extra={"spider": spider},)- assert spider.crawler.request_fingerprinterself._fingerprinter = spider.crawler.request_fingerprinterdef close_spider(self, spider: Spider) -> None:diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.pyindex 6c94d947e..b818569ba 100644--- a/scrapy/extensions/logstats.py+++ b/scrapy/extensions/logstats.py@@ -37,7 +37,6 @@ class LogStats:interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")if not interval:raise NotConfigured- assert crawler.statso = cls(crawler.stats, interval)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.pyindex 1fde6b296..35ff90e3b 100644--- a/scrapy/extensions/memdebug.py+++ b/scrapy/extensions/memdebug.py@@ -29,7 +29,6 @@ class MemoryDebugger:def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("MEMDEBUG_ENABLED"):raise NotConfigured- assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)return odiff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex e0e289ce8..ec761cfbf 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:from typing_extensions import Selffrom scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollectorlogger = logging.getLogger(__name__)@@ -43,6 +44,7 @@ class MemoryUsage:raise NotConfigured from excself.crawler: Crawler = crawler+ self._stats: StatsCollector = crawler.statsself.warned: bool = Falseself.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")if self.notify_mails: # pragma: no cover@@ -77,8 +79,7 @@ class MemoryUsage:return sizedef engine_started(self) -> None:- assert self.crawler.stats- self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())+ self._stats.set_value("memusage/startup", self.get_virtual_size())self.tasks: list[AsyncioLoopingCall | LoopingCall] = []tsk = create_looping_call(self.update)self.tasks.append(tsk)@@ -98,15 +99,12 @@ class MemoryUsage:tsk.stop()def update(self) -> None:- assert self.crawler.stats- self.crawler.stats.max_value("memusage/max", self.get_virtual_size())+ self._stats.max_value("memusage/max", self.get_virtual_size())def _check_limit(self) -> None:- assert self.crawler.engine- assert self.crawler.statspeak_mem_usage = self.get_virtual_size()if peak_mem_usage > self.limit:- self.crawler.stats.set_value("memusage/limit_reached", 1)+ self._stats.set_value("memusage/limit_reached", 1)mem = self.limit / 1024 / 1024logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",@@ -119,7 +117,7 @@ class MemoryUsage:f"memory usage exceeded {mem}MiB at {socket.gethostname()}")self._send_report(self.notify_mails, subj)- self.crawler.stats.set_value("memusage/limit_notified", 1)+ self._stats.set_value("memusage/limit_notified", 1)if self.crawler.engine.spider is not None:_schedule_coro(@@ -136,9 +134,8 @@ class MemoryUsage:def _check_warning(self) -> None:if self.warned: # warn only oncereturn- assert self.crawler.statsif self.get_virtual_size() > self.warning:- self.crawler.stats.set_value("memusage/warning_reached", 1)+ self._stats.set_value("memusage/warning_reached", 1)self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached)mem = self.warning / 1024 / 1024logger.warning(@@ -152,16 +149,13 @@ class MemoryUsage:f"memory usage reached {mem}MiB at {socket.gethostname()}")self._send_report(self.notify_mails, subj)- self.crawler.stats.set_value("memusage/warning_notified", 1)+ self._stats.set_value("memusage/warning_notified", 1)self.warned = Truedef _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover"""send notification mail with some additional useful info"""- assert self.crawler.engine- assert self.crawler.stats- stats = self.crawler.stats- s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"- s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"+ s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"+ s += f"Maximum memory usage : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n"s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"s += (diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.pyindex cbcc8b70e..adffbcbc4 100644--- a/scrapy/extensions/periodic_log.py+++ b/scrapy/extensions/periodic_log.py@@ -87,7 +87,6 @@ class PeriodicLog:)if not (ext_stats or ext_delta or ext_timing_enabled):raise NotConfigured- assert crawler.statsassert ext_stats is not Noneassert ext_delta is not Noneo = cls(diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.pyindex f05595806..7647cf33d 100644--- a/scrapy/extensions/statsmailer.py+++ b/scrapy/extensions/statsmailer.py@@ -42,7 +42,6 @@ class StatsMailer:if not recipients:raise NotConfiguredmail: MailSender = MailSender.from_crawler(crawler)- assert crawler.statso = cls(crawler.stats, recipients, mail)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)return odiff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 1506cb1ea..392f79299 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -108,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory):def _get_telnet_vars(self) -> dict[str, Any]:# Note: if you add entries here also update topics/telnetconsole.rst- assert self.crawler.enginetelnet_vars: dict[str, Any] = {"engine": self.crawler.engine,"spider": self.crawler.engine.spider,diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex cde73f12e..f44aee03d 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -45,7 +45,6 @@ class AutoThrottle:def _spider_opened(self, spider: Spider) -> None:self.mindelay = self._min_delay()self.maxdelay = self._max_delay()- assert self.crawler.engineself.crawler.engine.downloader._delay = self._start_delay()def _min_delay(self) -> float:@@ -98,7 +97,6 @@ class AutoThrottle:key: str | None = request.meta.get("download_slot")if key is None:return None, None- assert self.crawler.enginereturn key, self.crawler.engine.downloader.slots.get(key)def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 55a3676e5..e666f4ddd 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -697,9 +697,9 @@ class FilesPipeline(MediaPipeline):}def inc_stats(self, status: str) -> None:- assert self.crawler.stats- self.crawler.stats.inc_value("file_count")- self.crawler.stats.inc_value(f"file_status_count/{status}")+ stats = self.crawler.stats+ stats.inc_value("file_count")+ stats.inc_value(f"file_status_count/{status}")async def _file_downloaded(self,diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 764f82a78..d4025bf3b 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -100,7 +100,6 @@ class MediaPipeline(ABC):stacklevel=2,)self.crawler: Crawler = crawler- assert crawler.request_fingerprinterself._fingerprinter: RequestFingerprinterProtocol = (crawler.request_fingerprinter)@@ -228,7 +227,6 @@ class MediaPipeline(ABC):) -> FileInfo:try:self._modify_media_request(request)- assert self.crawler.engineresponse = await self.crawler.engine.download_async(request)return await ensure_awaitable(self.media_downloaded(response, request, info, item=item)diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 8e9783f5a..f4c35c0fe 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -265,7 +265,6 @@ class ScrapyPriorityQueue:class DownloaderInterface:def __init__(self, crawler: Crawler):- assert crawler.engineself.downloader: Downloader = crawler.engine.downloaderdef stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]:diff --git a/scrapy/shell.py b/scrapy/shell.pyindex dfea00c46..8f78af571 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -193,7 +193,6 @@ class Shell:"""if not self.spider:await self._open_spider(spider)- assert self.crawler.engine is not None# send the request to the engineself.crawler.engine.crawl(request)# this will fire when the request callback runs (via the callback hijacking in _request_deferred())@@ -204,7 +203,6 @@ class Shell:spider = self.crawler.spider or self.crawler._create_spider()self.crawler.spider = spider- assert self.crawler.engineawait self.crawler.engine.open_spider_async(close_if_idle=False)self.spider = spiderdiff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 1781543e3..49683168e 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -70,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware):maxdepth = settings.getint("DEPTH_LIMIT")verbose = settings.getbool("DEPTH_STATS_VERBOSE")prio = settings.getint("DEPTH_PRIORITY")- assert crawler.statso = cls(maxdepth, crawler.stats, verbose, prio)o.crawler = crawlerreturn odiff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 156b73e7e..116a02733 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -78,9 +78,9 @@ class HttpErrorMiddleware:self, response: Response, exception: Exception, spider: Spider | None = None) -> Iterable[Any] | None:if isinstance(exception, HttpError):- assert self.crawler.stats- self.crawler.stats.inc_value("httperror/response_ignored_count")- self.crawler.stats.inc_value(+ stats = self.crawler.stats+ stats.inc_value("httperror/response_ignored_count")+ stats.inc_value(f"httperror/response_ignored_status_count/{response.status}")logger.info(diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex f325ce7a0..86bdd2ed6 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware):{"maxlength": self.maxlength, "url": request.url},extra={"spider": self.crawler.spider},)- assert self.crawler.statsself.crawler.stats.inc_value("urllength/request_ignored_count")return Nonediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex ee357c031..bfa39169f 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -239,7 +239,6 @@ class LogCounterHandler(logging.Handler):def emit(self, record: logging.LogRecord) -> None:sname = f"log_count/{record.levelname}"- assert self.crawler.statsself.crawler.stats.inc_value(sname)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 358f20ed7..17ca02dea 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -74,6 +74,31 @@ class TestCrawler:assert not settings.frozenassert crawler.settings.frozen+ @pytest.mark.parametrize(+ "attr",+ ["extensions", "logformatter", "request_fingerprinter", "stats"],+ )+ def test_late_attr_before_apply_settings(self, attr: str) -> None:+ crawler = get_raw_crawler(DefaultSpider)+ with pytest.raises(RuntimeError, match=rf"Crawler\.{attr} is not set yet"):+ getattr(crawler, attr)+ crawler._apply_settings()+ assert getattr(crawler, attr) is not None++ @pytest.mark.parametrize(+ "attr",+ ["engine", "extensions", "logformatter", "request_fingerprinter", "stats"],+ )+ def test_late_attr_on_class(self, attr: str) -> None:+ # Introspection tools such as help() read these off the class.+ assert getattr(Crawler, attr) is getattr(Crawler, attr)++ def test_late_attr_engine_before_crawl(self) -> None:+ crawler = get_raw_crawler(DefaultSpider)+ crawler._apply_settings()+ with pytest.raises(RuntimeError, match=r"Crawler\.engine is not set yet"):+ _ = crawler.engine+@pytest.mark.parametrize(("attr", "setting"),[