Scrapy, a fast high-level web crawling & scraping framework for Python.
Expose the scheduler as a public engine property (#8099)
Adrian53eb8d60bcd0
diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex d6e8a54f9..2d6005068 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -273,4 +273,4 @@ Engine API==========.. autoclass:: scrapy.core.engine.ExecutionEngine()- :members: needs_backout+ :members: needs_backout, schedulerdiff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rstindex 89188e50c..caa06c306 100644--- a/docs/topics/optimize.rst+++ b/docs/topics/optimize.rst@@ -39,7 +39,7 @@ what every part of the engine is doing at a given moment:.. code-block:: textlen(engine.downloader.active) : 16- len(engine._slot.scheduler.mqs) : 92+ len(engine.scheduler.mqs) : 92len(engine.scraper.slot.active) : 0engine.scraper.slot.active_size : 0engine.scraper.slot.needs_backout() : False@@ -66,9 +66,9 @@ Take a few readings at different points of the crawl:callbacks and :ref:`item pipelines <topics-item-pipeline>` handle them. Thebottleneck is your own code.-- ``len(engine._slot.scheduler.mqs)`` grows without settling: the crawl- discovers requests faster than it downloads them. This is what makes long- crawls run out of memory.+- ``len(engine.scheduler.mqs)`` grows without settling: the crawl discovers+ requests faster than it downloads them. This is what makes long crawls run+ out of memory.Reading resource usage@@ -92,9 +92,9 @@ Memoryfar above :stat:`memusage/startup` is expected; what matters is whether itkeeps growing for as long as the crawl runs.- Growth that tracks ``len(engine._slot.scheduler.mqs)`` is a scheduling- problem, covered in :ref:`optimize-memory`. Growth that does not is a- :ref:`memory leak <topics-leaks>`.+ Growth that tracks ``len(engine.scheduler.mqs)`` is a scheduling problem,+ covered in :ref:`optimize-memory`. Growth that does not is a :ref:`memory+ leak <topics-leaks>`.NetworkCompare :stat:`downloader/response_bytes` over the crawl time against yourdiff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rstindex 54379b4b9..6448121ab 100644--- a/docs/topics/telnetconsole.rst+++ b/docs/topics/telnetconsole.rst@@ -121,8 +121,8 @@ engine status::engine.spider_is_idle() : Falseengine._slot.closing : Falselen(engine._slot.inprogress) : 16- len(engine._slot.scheduler.dqs or []) : 0- len(engine._slot.scheduler.mqs) : 92+ len(engine.scheduler.dqs or []) : 0+ len(engine.scheduler.mqs) : 92len(engine.scraper.slot.queue) : 0len(engine.scraper.slot.active) : 0engine.scraper.slot.active_size : 0diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 83e3381d6..0764a5812 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -163,6 +163,14 @@ class ExecutionEngine:)return scheduler_cls+ @property+ def scheduler(self) -> BaseScheduler | None:+ """The scheduler in use, or ``None`` before the spider has started.++ .. versionadded:: VERSION+ """+ return self._slot.scheduler if self._slot is not None else None+def start(self, _start_request_processing: bool = True) -> Deferred[None]: # pragma: no coverdiff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex 085720f66..687739bbb 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -20,8 +20,8 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:"engine.spider_is_idle()","engine._slot.closing","len(engine._slot.inprogress)",- "len(engine._slot.scheduler.dqs or [])",- "len(engine._slot.scheduler.mqs)",+ "len(engine.scheduler.dqs or [])",+ "len(engine.scheduler.mqs)","len(engine.scraper.slot.queue)","len(engine.scraper.slot.active)","engine.scraper.slot.active_size",