Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type late Crawler attributes as always set instead of None (#7882)

Adrian508bd7faec44
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 7ff8d3464..598edfeb5 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
+ The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`,
+ :attr:`request_fingerprinter` and :attr:`stats` attributes get their value
+ when the crawl starts, and raise :exc:`RuntimeError` when read before that.
+
+ .. versionchanged:: VERSION
+ Those attributes used to be ``None`` before getting their value.
+
.. attribute:: request_fingerprinter
The request fingerprint builder of this crawler.
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 51caed57f..b6255ec7e 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand):
) -> list[Any]:
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
- assert self.pcrawler.engine
itemproc = self.pcrawler.engine.scraper.itemproc
if hasattr(itemproc, "process_item_async"):
for item in items:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b7a1e0fcd..b63d136a4 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -112,7 +112,6 @@ class ExecutionEngine:
self.crawler: Crawler = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
- assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
self._slot: _Slot | None = None
self.spider: Spider | None = None
@@ -299,7 +298,6 @@ class ExecutionEngine:
response=None,
spider=self.spider,
)
- assert self.crawler.stats
self.crawler.stats.inc_value("spider_exceptions/count")
self.crawler.stats.inc_value(
f"spider_exceptions/{type(exception).__name__}"
@@ -563,17 +561,17 @@ class ExecutionEngine:
if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
await maybe_deferred_to_future(d)
await self.scraper.open_spider_async()
- assert self.crawler.stats
- if argument_is_required(self.crawler.stats.open_spider, "spider"):
+ stats = self.crawler.stats
+ if argument_is_required(stats.open_spider, "spider"):
warnings.warn(
- f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+ f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"
f" this is deprecated and the argument will not be passed in future Scrapy versions.",
ScrapyDeprecationWarning,
stacklevel=2,
)
- self.crawler.stats.open_spider(spider=self.crawler.spider)
+ stats.open_spider(spider=self.crawler.spider)
else:
- self.crawler.stats.open_spider()
+ stats.open_spider()
await self.signals.send_catch_log_async(
signals.spider_opened, spider=self.crawler.spider
)
@@ -676,20 +674,18 @@ class ExecutionEngine:
extra={"spider": spider},
)
- assert self.crawler.stats
try:
- if argument_is_required(self.crawler.stats.close_spider, "spider"):
+ stats = self.crawler.stats
+ if argument_is_required(stats.close_spider, "spider"):
warnings.warn(
- f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+ f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument,"
f" this is deprecated and the argument will not be passed in future Scrapy versions.",
ScrapyDeprecationWarning,
stacklevel=2,
)
- self.crawler.stats.close_spider(
- spider=self.crawler.spider, reason=reason
- )
+ stats.close_spider(spider=self.crawler.spider, reason=reason)
else:
- self.crawler.stats.close_spider(reason=reason)
+ stats.close_spider(reason=reason)
except Exception:
logger.error("Stats close failure")
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 58e37ce5e..6426b1751 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -120,7 +120,6 @@ class Scraper:
self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler: Crawler = crawler
self.signals: SignalManager = crawler.signals
- assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
def _check_deprecated_itemproc_method(self, method: str) -> None:
@@ -355,7 +354,6 @@ class Scraper:
assert self.crawler.spider
exc = _failure.value
if isinstance(exc, CloseSpider):
- assert self.crawler.engine is not None # typing
_schedule_coro(
self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
)
@@ -374,11 +372,9 @@ class Scraper:
response=response,
spider=self.crawler.spider,
)
- assert self.crawler.stats
- self.crawler.stats.inc_value("spider_exceptions/count")
- self.crawler.stats.inc_value(
- f"spider_exceptions/{_failure.value.__class__.__name__}"
- )
+ stats = self.crawler.stats
+ stats.inc_value("spider_exceptions/count")
+ stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")
def handle_spider_output(
self,
@@ -456,7 +452,6 @@ class Scraper:
Items are sent to the item pipelines, requests are scheduled.
"""
if isinstance(output, Request):
- assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
return
if output is not None:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e2f726519..44c4ffdcf 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -8,7 +8,7 @@ import signal
import warnings
from abc import ABC, abstractmethod
from functools import partial
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@@ -58,7 +58,55 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
+class _LateAttribute(Generic[_T]):
+ """Descriptor for a :class:`Crawler` attribute that only gets a value once
+ the crawl starts.
+
+ The value is kept in an attribute of the same name prefixed with an
+ underscore, and reading it before it is set raises :exc:`RuntimeError`.
+ This way the public attribute can be annotated as always set, and its
+ users, both in Scrapy and in third-party code, do not need to narrow its
+ type on every use. Code that runs before the crawl starts reads the
+ underscore-prefixed attribute instead.
+ """
+
+ def __set_name__(self, owner: type[Crawler], name: str) -> None:
+ self._name = name
+ self._private_name = f"_{name}"
+
+ @overload
+ def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ...
+
+ @overload
+ def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ...
+
+ def __get__(
+ self, instance: Crawler | None, owner: type[Crawler]
+ ) -> _LateAttribute[_T] | _T:
+ if instance is None:
+ return self
+ value: _T | None = getattr(instance, self._private_name)
+ if value is None:
+ raise RuntimeError(
+ f"Crawler.{self._name} is not set yet. It is set when the "
+ "crawl starts, so it can only be used from then on, e.g. "
+ "from the spider_opened signal handler onwards."
+ )
+ return value
+
+ def __set__(self, instance: Crawler, value: _T) -> None:
+ setattr(instance, self._private_name, value)
+
+
class Crawler:
+ engine: _LateAttribute[ExecutionEngine] = _LateAttribute()
+ extensions: _LateAttribute[ExtensionManager] = _LateAttribute()
+ logformatter: _LateAttribute[LogFormatter] = _LateAttribute()
+ request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = (
+ _LateAttribute()
+ )
+ stats: _LateAttribute[StatsCollector] = _LateAttribute()
+
def __init__(
self,
spidercls: type[Spider],
@@ -83,12 +131,13 @@ class Crawler:
self.crawling: bool = False
self._started: bool = False
- self.extensions: ExtensionManager | None = None
- self.stats: StatsCollector | None = None
- self.logformatter: LogFormatter | None = None
- self.request_fingerprinter: RequestFingerprinterProtocol | None = None
self.spider: Spider | None = None
- self.engine: ExecutionEngine | None = None
+
+ self._engine: ExecutionEngine | None = None
+ self._extensions: ExtensionManager | None = None
+ self._logformatter: LogFormatter | None = None
+ self._request_fingerprinter: RequestFingerprinterProtocol | None = None
+ self._stats: StatsCollector | None = None
def _update_root_log_handler(self) -> None:
if get_scrapy_root_handler() is not None:
@@ -225,8 +274,8 @@ class Crawler:
yield deferred_from_coro(self.engine.start_async())
except Exception:
self.crawling = False
- if self.engine is not None:
- yield deferred_from_coro(self.engine.close_async())
+ if self._engine is not None:
+ yield deferred_from_coro(self._engine.close_async())
raise
async def crawl_async(self, *args: Any, **kwargs: Any) -> None:
@@ -255,8 +304,8 @@ class Crawler:
await self.engine.start_async()
except Exception:
self.crawling = False
- if self.engine is not None:
- await self.engine.close_async()
+ if self._engine is not None:
+ await self._engine.close_async()
raise
def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
@@ -282,7 +331,6 @@ class Crawler:
"""
if self.crawling:
self.crawling = False
- assert self.engine
if self.engine.running:
await self.engine.stop_async()
@@ -313,7 +361,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
- if not self.engine:
+ if self._engine is None:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
"the crawl engine has been created."
@@ -331,7 +379,7 @@ class Crawler:
created, e.g. at signals :signal:`engine_started` or
:signal:`spider_opened`.
"""
- if not self.extensions:
+ if self._extensions is None:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
"extension manager has been created."
@@ -348,7 +396,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
- if not self.engine:
+ if self._engine is None:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
"crawl engine has been created."
@@ -365,7 +413,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
- if not self.engine:
+ if self._engine is None:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
"crawl engine has been created."
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index e7ca0ac0e..3ebd98027 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -55,7 +55,6 @@ class HttpCacheMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- assert crawler.stats
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 0d5c9e4c9..b9a26df3a 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -61,7 +61,6 @@ class OffsiteMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index f910d07c8..dd2897ee4 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -94,7 +94,6 @@ def get_retry_request(
retry-related job stats
"""
settings = spider.crawler.settings
- assert spider.crawler.stats
stats = spider.crawler.stats
retry_times = request.meta.get("retry_times", 0) + 1
if max_retry_times is None:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 81a3a887f..d7aa8738c 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.robotstxt import RobotParser
+ from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
@@ -41,6 +42,7 @@ class RobotsTxtMiddleware:
self._default_useragent: str = crawler.settings["USER_AGENT"]
self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
self.crawler: Crawler = crawler
+ self._stats: StatsCollector = crawler.stats
self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
self._parserimpl: RobotParser = load_object(
crawler.settings.get("ROBOTSTXT_PARSER")
@@ -78,8 +80,7 @@ class RobotsTxtMiddleware:
{"request": request},
extra={"spider": self.crawler.spider},
)
- assert self.crawler.stats
- self.crawler.stats.inc_value("robotstxt/forbidden")
+ self._stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
async def robot_parser(self, request: Request) -> RobotParser | None:
@@ -95,8 +96,6 @@ class RobotsTxtMiddleware:
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
- assert self.crawler.engine
- assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
await self._parse_robots(resp, netloc, request)
@@ -109,7 +108,7 @@ class RobotsTxtMiddleware:
extra={"spider": self.crawler.spider},
)
self._robots_error(e, netloc)
- self.crawler.stats.inc_value("robotstxt/request_count")
+ self._stats.inc_value("robotstxt/request_count")
parser = self._parsers[netloc]
if isinstance(parser, Deferred):
@@ -119,11 +118,8 @@ class RobotsTxtMiddleware:
async def _parse_robots(
self, response: Response, netloc: str, request: Request
) -> None:
- assert self.crawler.stats
- self.crawler.stats.inc_value("robotstxt/response_count")
- self.crawler.stats.inc_value(
- f"robotstxt/response_status_count/{response.status}"
- )
+ self._stats.inc_value("robotstxt/response_count")
+ self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")
rp = self._parserimpl.from_crawler(self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
@@ -138,8 +134,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, exc: Exception, netloc: str) -> None:
if not isinstance(exc, IgnoreRequest):
key = f"robotstxt/exception_count/{type(exc)}"
- assert self.crawler.stats
- self.crawler.stats.inc_value(key)
+ self._stats.inc_value(key)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = None
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index bafa931de..07de1c2e7 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -43,7 +43,6 @@ class DownloaderStats:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
- assert crawler.stats
return cls(crawler.stats)
@_warn_spider_arg
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 36fb0f97d..09f0be63b 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter):
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- assert crawler.request_fingerprinter
debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
return cls(
job_dir(crawler.settings),
@@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter):
self.logger.debug(msg, {"request": request}, extra={"spider": spider})
self.logdupes = False
- assert spider.crawler.stats
spider.crawler.stats.inc_value("dupefilter/filtered")
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 9cb792e30..5d40e5f9f 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -102,7 +102,6 @@ class CloseSpider:
self._close_spider("closespider_pagecount_no_item")
def spider_opened(self, spider: Spider) -> None:
- assert self.crawler.engine
self.task = call_later(
self.close_on["timeout"], self._close_spider, "closespider_timeout"
)
@@ -146,5 +145,4 @@ class CloseSpider:
self._close_spider("closespider_timeout_no_item")
def _close_spider(self, reason: str) -> None:
- assert self.crawler.engine
_schedule_coro(self.crawler.engine.close_spider_async(reason=reason))
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 6a5e55992..b464942af 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -26,7 +26,6 @@ class CoreStats:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 5def7509e..8802a3b58 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -45,7 +45,6 @@ class StackTraceDump:
return cls(crawler)
def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None:
- assert self.crawler.engine
log_args = {
"stackdumps": self._thread_stacks(),
"enginestatus": format_engine_status(self.crawler.engine),
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e4294ac90..32abc82d3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -611,7 +611,6 @@ class FeedExporter:
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
slot_type = type(slot.storage).__name__
- assert self.crawler.stats
try:
await ensure_awaitable(slot.storage.store(self._get_file(slot)))
except Exception:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index dbb79b02d..d008d0f67 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -261,7 +261,6 @@ class DbmCacheStorage:
extra={"spider": spider},
)
- assert spider.crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinterProtocol = (
spider.crawler.request_fingerprinter
)
@@ -326,7 +325,6 @@ class FilesystemCacheStorage:
extra={"spider": spider},
)
- assert spider.crawler.request_fingerprinter
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider: Spider) -> None:
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 6c94d947e..b818569ba 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -37,7 +37,6 @@ class LogStats:
interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
- assert crawler.stats
o = cls(crawler.stats, interval)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 1fde6b296..35ff90e3b 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -29,7 +29,6 @@ class MemoryDebugger:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
raise NotConfigured
- assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index e0e289ce8..ec761cfbf 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
@@ -43,6 +44,7 @@ class MemoryUsage:
raise NotConfigured from exc
self.crawler: Crawler = crawler
+ self._stats: StatsCollector = crawler.stats
self.warned: bool = False
self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
if self.notify_mails: # pragma: no cover
@@ -77,8 +79,7 @@ class MemoryUsage:
return size
def engine_started(self) -> None:
- assert self.crawler.stats
- self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
+ self._stats.set_value("memusage/startup", self.get_virtual_size())
self.tasks: list[AsyncioLoopingCall | LoopingCall] = []
tsk = create_looping_call(self.update)
self.tasks.append(tsk)
@@ -98,15 +99,12 @@ class MemoryUsage:
tsk.stop()
def update(self) -> None:
- assert self.crawler.stats
- self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
+ self._stats.max_value("memusage/max", self.get_virtual_size())
def _check_limit(self) -> None:
- assert self.crawler.engine
- assert self.crawler.stats
peak_mem_usage = self.get_virtual_size()
if peak_mem_usage > self.limit:
- self.crawler.stats.set_value("memusage/limit_reached", 1)
+ self._stats.set_value("memusage/limit_reached", 1)
mem = self.limit / 1024 / 1024
logger.error(
"Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
@@ -119,7 +117,7 @@ class MemoryUsage:
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
- self.crawler.stats.set_value("memusage/limit_notified", 1)
+ self._stats.set_value("memusage/limit_notified", 1)
if self.crawler.engine.spider is not None:
_schedule_coro(
@@ -136,9 +134,8 @@ class MemoryUsage:
def _check_warning(self) -> None:
if self.warned: # warn only once
return
- assert self.crawler.stats
if self.get_virtual_size() > self.warning:
- self.crawler.stats.set_value("memusage/warning_reached", 1)
+ self._stats.set_value("memusage/warning_reached", 1)
self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached)
mem = self.warning / 1024 / 1024
logger.warning(
@@ -152,16 +149,13 @@ class MemoryUsage:
f"memory usage reached {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
- self.crawler.stats.set_value("memusage/warning_notified", 1)
+ self._stats.set_value("memusage/warning_notified", 1)
self.warned = True
def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover
"""send notification mail with some additional useful info"""
- assert self.crawler.engine
- assert self.crawler.stats
- stats = self.crawler.stats
- s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
- s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
+ s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
+ s += f"Maximum memory usage : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
s += (
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index cbcc8b70e..adffbcbc4 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -87,7 +87,6 @@ class PeriodicLog:
)
if not (ext_stats or ext_delta or ext_timing_enabled):
raise NotConfigured
- assert crawler.stats
assert ext_stats is not None
assert ext_delta is not None
o = cls(
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index f05595806..7647cf33d 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -42,7 +42,6 @@ class StatsMailer:
if not recipients:
raise NotConfigured
mail: MailSender = MailSender.from_crawler(crawler)
- assert crawler.stats
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 1506cb1ea..392f79299 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -108,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory):
def _get_telnet_vars(self) -> dict[str, Any]:
# Note: if you add entries here also update topics/telnetconsole.rst
- assert self.crawler.engine
telnet_vars: dict[str, Any] = {
"engine": self.crawler.engine,
"spider": self.crawler.engine.spider,
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index cde73f12e..f44aee03d 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -45,7 +45,6 @@ class AutoThrottle:
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay()
self.maxdelay = self._max_delay()
- assert self.crawler.engine
self.crawler.engine.downloader._delay = self._start_delay()
def _min_delay(self) -> float:
@@ -98,7 +97,6 @@ class AutoThrottle:
key: str | None = request.meta.get("download_slot")
if key is None:
return None, None
- assert self.crawler.engine
return key, self.crawler.engine.downloader.slots.get(key)
def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 55a3676e5..e666f4ddd 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -697,9 +697,9 @@ class FilesPipeline(MediaPipeline):
}
def inc_stats(self, status: str) -> None:
- assert self.crawler.stats
- self.crawler.stats.inc_value("file_count")
- self.crawler.stats.inc_value(f"file_status_count/{status}")
+ stats = self.crawler.stats
+ stats.inc_value("file_count")
+ stats.inc_value(f"file_status_count/{status}")
async def _file_downloaded(
self,
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 764f82a78..d4025bf3b 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -100,7 +100,6 @@ class MediaPipeline(ABC):
stacklevel=2,
)
self.crawler: Crawler = crawler
- assert crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinterProtocol = (
crawler.request_fingerprinter
)
@@ -228,7 +227,6 @@ class MediaPipeline(ABC):
) -> FileInfo:
try:
self._modify_media_request(request)
- assert self.crawler.engine
response = await self.crawler.engine.download_async(request)
return await ensure_awaitable(
self.media_downloaded(response, request, info, item=item)
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 8e9783f5a..f4c35c0fe 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -265,7 +265,6 @@ class ScrapyPriorityQueue:
class DownloaderInterface:
def __init__(self, crawler: Crawler):
- assert crawler.engine
self.downloader: Downloader = crawler.engine.downloader
def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]:
diff --git a/scrapy/shell.py b/scrapy/shell.py
index dfea00c46..8f78af571 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -193,7 +193,6 @@ class Shell:
"""
if not self.spider:
await self._open_spider(spider)
- assert self.crawler.engine is not None
# send the request to the engine
self.crawler.engine.crawl(request)
# this will fire when the request callback runs (via the callback hijacking in _request_deferred())
@@ -204,7 +203,6 @@ class Shell:
spider = self.crawler.spider or self.crawler._create_spider()
self.crawler.spider = spider
- assert self.crawler.engine
await self.crawler.engine.open_spider_async(close_if_idle=False)
self.spider = spider
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 1781543e3..49683168e 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -70,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware):
maxdepth = settings.getint("DEPTH_LIMIT")
verbose = settings.getbool("DEPTH_STATS_VERBOSE")
prio = settings.getint("DEPTH_PRIORITY")
- assert crawler.stats
o = cls(maxdepth, crawler.stats, verbose, prio)
o.crawler = crawler
return o
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 156b73e7e..116a02733 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -78,9 +78,9 @@ class HttpErrorMiddleware:
self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
if isinstance(exception, HttpError):
- assert self.crawler.stats
- self.crawler.stats.inc_value("httperror/response_ignored_count")
- self.crawler.stats.inc_value(
+ stats = self.crawler.stats
+ stats.inc_value("httperror/response_ignored_count")
+ stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}"
)
logger.info(
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index f325ce7a0..86bdd2ed6 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware):
{"maxlength": self.maxlength, "url": request.url},
extra={"spider": self.crawler.spider},
)
- assert self.crawler.stats
self.crawler.stats.inc_value("urllength/request_ignored_count")
return None
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index ee357c031..bfa39169f 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -239,7 +239,6 @@ class LogCounterHandler(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
sname = f"log_count/{record.levelname}"
- assert self.crawler.stats
self.crawler.stats.inc_value(sname)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 358f20ed7..17ca02dea 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -74,6 +74,31 @@ class TestCrawler:
assert not settings.frozen
assert crawler.settings.frozen
+ @pytest.mark.parametrize(
+ "attr",
+ ["extensions", "logformatter", "request_fingerprinter", "stats"],
+ )
+ def test_late_attr_before_apply_settings(self, attr: str) -> None:
+ crawler = get_raw_crawler(DefaultSpider)
+ with pytest.raises(RuntimeError, match=rf"Crawler\.{attr} is not set yet"):
+ getattr(crawler, attr)
+ crawler._apply_settings()
+ assert getattr(crawler, attr) is not None
+
+ @pytest.mark.parametrize(
+ "attr",
+ ["engine", "extensions", "logformatter", "request_fingerprinter", "stats"],
+ )
+ def test_late_attr_on_class(self, attr: str) -> None:
+ # Introspection tools such as help() read these off the class.
+ assert getattr(Crawler, attr) is getattr(Crawler, attr)
+
+ def test_late_attr_engine_before_crawl(self) -> None:
+ crawler = get_raw_crawler(DefaultSpider)
+ crawler._apply_settings()
+ with pytest.raises(RuntimeError, match=r"Crawler\.engine is not set yet"):
+ _ = crawler.engine
+
@pytest.mark.parametrize(
("attr", "setting"),
[