Scrapy, a fast high-level web crawling & scraping framework for Python.
Make ExecutionEngine a state machine. (#8181)
Andrey Rakhmatullin5ecc3f5023cd
diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex 4f87c2041..4e3ea5a61 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -215,4 +215,4 @@ Engine API==========.. autoclass:: scrapy.core.engine.ExecutionEngine()- :members: needs_backout, scheduler+ :members: needs_backout, running, schedulerdiff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 1ac33bea9..04792f655 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -11,6 +11,7 @@ import asyncioimport contextlibimport loggingimport warnings+from enum import Enumfrom functools import partialfrom time import timefrom traceback import format_exc@@ -64,6 +65,43 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+class _EngineState(Enum):+ """The lifecycle state of ExecutionEngine.++ The engine goes through these states in order, possibly skipping some."""++ # The engine has been created but no spider has been opened yet.+ CREATED = "created"+ # open_spider_async() is running.+ SPIDER_OPENING = "spider_opening"+ # The spider is open but start_async() has not been called yet.+ SPIDER_OPEN = "spider_open"+ # start_async() is running, engine_started may be in flight.+ STARTING = "starting"+ # The engine is running.+ RUNNING = "running"+ # stop_async() is running, engine_stopped has not been sent yet.+ STOPPING = "stopping"+ # The engine has been stopped and cannot be reused.+ STOPPED = "stopped"+++_STATE_TRANSITIONS: dict[_EngineState, frozenset[_EngineState]] = {+ # Shortcut to STOPPED: close_async() without a spider+ _EngineState.CREATED: frozenset(+ {_EngineState.SPIDER_OPENING, _EngineState.STOPPED}+ ),+ _EngineState.SPIDER_OPENING: frozenset({_EngineState.SPIDER_OPEN}),+ # Shortcut to STOPPED: close_async() before start_async()+ _EngineState.SPIDER_OPEN: frozenset({_EngineState.STARTING, _EngineState.STOPPED}),+ # Shortcut to STOPPING: stop_async() from an engine_started handler+ _EngineState.STARTING: frozenset({_EngineState.RUNNING, _EngineState.STOPPING}),+ _EngineState.RUNNING: frozenset({_EngineState.STOPPING}),+ _EngineState.STOPPING: frozenset({_EngineState.STOPPED}),+ _EngineState.STOPPED: frozenset(),+}++class _Slot:def __init__(self,@@ -116,9 +154,7 @@ class ExecutionEngine:self.logformatter: LogFormatter = crawler.logformatterself._slot: _Slot | None = Noneself.spider: Spider | None = None- self.running: bool = False- self._starting: bool = False- self._stopping: bool = False+ self._state: _EngineState = _EngineState.CREATEDself._stop_mode: _StopMode = "graceful"self._downloader_fast_stopped: bool = Falseself.paused: bool = False@@ -174,6 +210,37 @@ class ExecutionEngine:"""return self._slot.scheduler if self._slot is not None else None+ @property+ def running(self) -> bool:+ """Whether the engine is running, i.e. :meth:`start_async` has started+ processing requests and :meth:`stop_async` has not been called yet.++ .. versionchanged:: VERSION+ Became a read-only property.+ """+ return self._state is _EngineState.RUNNING++ @running.setter+ def running(self, value: bool) -> None:+ warnings.warn(+ "Setting ExecutionEngine.running is deprecated and has no effect.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )++ def _transition_to(self, state: _EngineState) -> None:+ """Set the state, validating the transition.++ If the transition is invalid, a warning is logged and the transition+ still happens.+ """+ if state not in _STATE_TRANSITIONS[self._state]:+ logger.warning(+ f"Invalid engine state transition: {self._state.name} → {state.name}",+ stack_info=True,+ )+ self._state = state+def start(self, _start_request_processing: bool = True) -> Deferred[None]: # pragma: no cover@@ -189,18 +256,23 @@ class ExecutionEngine:.. versionadded:: 2.14"""- if self._starting:+ if self._state not in (+ _EngineState.CREATED,+ _EngineState.SPIDER_OPENING,+ _EngineState.SPIDER_OPEN,+ ):raise RuntimeError("Engine already running")self.start_time = time()- self._starting = True+ self._transition_to(_EngineState.STARTING)await self.signals.send_catch_log_async(signal=signals.engine_started)- if self._stopping:- # band-aid until https://github.com/scrapy/scrapy/issues/6916+ if self._state in (_EngineState.STOPPING, _EngineState.STOPPED):+ # An engine_started handler stopped the engine.+ # Band-aid until the next stage of https://github.com/scrapy/scrapy/issues/6916returnif self.spider is None:# require an opened spiderreturn- self.running = True+ self._transition_to(_EngineState.RUNNING)self._closewait = Deferred()coro = self._start_request_processing()if is_asyncio_available():@@ -230,20 +302,23 @@ class ExecutionEngine:mode = _normalize_stop_mode(mode, allow_force=False)- if not self._starting and not self._stopping:+ if self._state in (+ _EngineState.CREATED,+ _EngineState.SPIDER_OPENING,+ _EngineState.SPIDER_OPEN,+ ):raise RuntimeError("Engine not running")self._stop_mode = _max_stop_mode(self._stop_mode, mode)- if self._stopping:+ if self._state in (_EngineState.STOPPING, _EngineState.STOPPED):if self.spider is not None and self._stop_mode == "fast":await self.close_spider_async(reason="shutdown", mode="fast")if self._closewait:await maybe_deferred_to_future(self._closewait)return- self.running = self._starting = False- self._stopping = True+ self._transition_to(_EngineState.STOPPING)if self._start_request_processing_awaitable is not None:if (not is_asyncio_available()@@ -261,6 +336,10 @@ class ExecutionEngine:# Normally it's close_spider_async() that does this.self.downloader.close()await self.signals.send_catch_log_async(signal=signals.engine_stopped)+ # Firing _closewait can resume start_async(), and with it the code+ # awaiting start_async(), before this method continues, so the engine+ # must already be STOPPED when that code checks the state.+ self._transition_to(_EngineState.STOPPED)if self._closewait:self._closewait.callback(None)@@ -279,10 +358,13 @@ class ExecutionEngine:"""if self.running:await self.stop_async() # will also close spider and downloader- elif self.spider is not None:+ return+ if self.spider is not None:await self.close_spider_async(reason=reason) # will also close downloaderelse:self.downloader.close()+ if self._state not in (_EngineState.STOPPING, _EngineState.STOPPED):+ self._transition_to(_EngineState.STOPPED)def pause(self) -> None:self.paused = True@@ -579,43 +661,49 @@ class ExecutionEngine:f"No free spider slot when opening {self.crawler.spider.name!r}")scheduler = build_from_crawler(self.scheduler_cls, self.crawler)+ self._transition_to(_EngineState.SPIDER_OPENING)logger.info("Spider opened", extra={"spider": self.crawler.spider})self.spider = self.crawler.spidernextcall = CallLaterOnce(self._start_scheduled_requests)self._slot = _Slot(close_if_idle, nextcall, scheduler)- # A component that fails to start can ask for the spider to be closed.- # The rest of the startup runs anyway, so that components that are- # started also get stopped, and the request is honored once the spider- # is open.- close_spider_exc: CloseSpider | None = Nonetry:- self._start = await self.scraper.spidermw.process_start()- if hasattr(scheduler, "open") and (- d := scheduler.open(self.crawler.spider)- ):- await maybe_deferred_to_future(d)- await self.scraper.open_spider_async()- except CloseSpider as exc:- close_spider_exc = exc- stats = self.crawler.stats- if argument_is_required(stats.open_spider, "spider"):- warnings.warn(- f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"- f" this is deprecated and the argument will not be passed in future Scrapy versions.",- ScrapyDeprecationWarning,- stacklevel=2,+ # A component that fails to start can ask for the spider to be closed.+ # The rest of the startup runs anyway, so that components that are+ # started also get stopped, and the request is honored once the spider+ # is open.+ close_spider_exc: CloseSpider | None = None+ try:+ self._start = await self.scraper.spidermw.process_start()+ if hasattr(scheduler, "open") and (+ d := scheduler.open(self.crawler.spider)+ ):+ await maybe_deferred_to_future(d)+ await self.scraper.open_spider_async()+ except CloseSpider as exc:+ close_spider_exc = exc+ stats = self.crawler.stats+ if argument_is_required(stats.open_spider, "spider"):+ warnings.warn(+ f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"+ f" this is deprecated and the argument will not be passed in future Scrapy versions.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ stats.open_spider(spider=self.crawler.spider)+ else:+ stats.open_spider()+ results = await self.signals.send_catch_log_async(+ signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider)- stats.open_spider(spider=self.crawler.spider)- else:- stats.open_spider()- results = await self.signals.send_catch_log_async(- signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider- )- for _, result in results:- if isinstance(result, CloseSpider):- close_spider_exc = close_spider_exc or result- if close_spider_exc is not None:- raise close_spider_exc+ for _, result in results:+ if isinstance(result, CloseSpider):+ close_spider_exc = close_spider_exc or result+ if close_spider_exc is not None:+ raise close_spider_exc+ finally:+ # Partially open counts as open because we still want to close the+ # components that were opened (close_spider_async() will do that).+ self._transition_to(_EngineState.SPIDER_OPEN)def _spider_idle(self) -> None:"""diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 14e58a975..bd4646ce9 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -13,7 +13,7 @@ from twisted.python.failure import Failurefrom scrapy import signalsfrom scrapy.core.downloader import Downloader-from scrapy.core.engine import ExecutionEngine, _Slot+from scrapy.core.engine import ExecutionEngine, _EngineState, _Slotfrom scrapy.core.scheduler import BaseSchedulerfrom scrapy.exceptions import (CloseSpider,@@ -167,7 +167,7 @@ class TestEngine(TestEngineBase):async def test_stop_async_reentrant_fast_waits_for_closewait(self) -> None:engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)engine.spider = Mock()- engine._stopping = True+ engine._state = _EngineState.STOPPINGengine._closewait = defer.Deferred()with patch.object(@@ -187,7 +187,7 @@ class TestEngine(TestEngineBase):self,) -> None:engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)- engine._stopping = True+ engine._state = _EngineState.STOPPINGwith patch.object(engine, "close_spider_async", new_callable=AsyncMock@@ -445,11 +445,16 @@ async def test_scheduler_creation_error() -> None:@coroutine_test-async def test_stop_without_spider_closes_downloader() -> None:+async def test_stop_without_spider_closes_downloader(+ caplog: pytest.LogCaptureFixture,+) -> None:crawler = get_crawler(DefaultSpider)engine = crawler.engine = ExecutionEngine(crawler, lambda _: None)engine.downloader.close = Mock(wraps=engine.downloader.close) # type: ignore[method-assign]- await engine.start_async()+ with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):+ await engine.start_async()+ # Starting without a spider is not part of the engine lifecycle.+ assert "Invalid engine state transition: CREATED → STARTING" in caplog.textawait engine.stop_async()engine.downloader.close.assert_called_once()diff --git a/tests/test_engine_lifecycle.py b/tests/test_engine_lifecycle.pynew file mode 100644index 000000000..919105023--- /dev/null+++ b/tests/test_engine_lifecycle.py@@ -0,0 +1,147 @@+from __future__ import annotations++import logging+from typing import TYPE_CHECKING, Any+from unittest.mock import Mock++import pytest++from scrapy import Request, signals+from scrapy.core.engine import ExecutionEngine, _EngineState+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from collections.abc import Callable++ from scrapy.crawler import Crawler+++class SignalRecorder:+ """Record lifecycle signals and the engine state at the time each signal+ fired."""++ SIGNALS = ("engine_started", "engine_stopped", "spider_opened", "spider_closed")++ def __init__(self, crawler: Crawler) -> None:+ self.crawler = crawler+ self.calls: list[tuple[str, _EngineState]] = []+ # Keep strong references to the handlers: signal connections are weak.+ self._handlers = [self._make_handler(name) for name in self.SIGNALS]+ for name, handler in zip(self.SIGNALS, self._handlers, strict=True):+ crawler.signals.connect(handler, getattr(signals, name))++ def _make_handler(self, name: str) -> Callable[..., None]:+ def handler(**kwargs: Any) -> None:+ assert self.crawler.engine is not None+ self.calls.append((name, self.crawler.engine._state))++ return handler++ @property+ def names(self) -> list[str]:+ return [call[0] for call in self.calls]++ @property+ def states(self) -> list[_EngineState]:+ return [call[1] for call in self.calls]+++def assert_state(engine: ExecutionEngine, state: _EngineState) -> None:+ assert engine._state is state+++def make_engine(crawler: Crawler) -> ExecutionEngine:+ """Create the engine of *crawler* as Crawler.crawl_async() (and scrapy+ shell) would, without opening or starting anything."""+ crawler.spider = crawler._create_spider()+ engine = crawler.engine = crawler._create_engine()+ return engine+++def assert_no_invalid_transition(caplog: pytest.LogCaptureFixture) -> None:+ assert not [+ r for r in caplog.records if "Invalid engine state transition" in r.message+ ]+++@coroutine_test+async def test_state_progression(caplog: pytest.LogCaptureFixture) -> None:+ crawler = get_crawler(DefaultSpider)+ recorder = SignalRecorder(crawler)+ with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):+ await crawler.crawl_async()+ assert_no_invalid_transition(caplog)+ engine = crawler.engine+ assert engine is not None+ assert_state(engine, _EngineState.STOPPED)+ assert not engine.running+ assert recorder.names == [+ "spider_opened",+ "engine_started",+ "spider_closed",+ "engine_stopped",+ ]+ assert recorder.states == [+ _EngineState.SPIDER_OPENING,+ _EngineState.STARTING,+ _EngineState.RUNNING,+ _EngineState.STOPPING,+ ]+++@coroutine_test+async def test_fetch_only_lifecycle(caplog: pytest.LogCaptureFixture) -> None:+ """The lifecycle used by scrapy shell: the spider is opened, requests are+ downloaded, and the engine is never started."""+ crawler = get_crawler(DefaultSpider)+ recorder = SignalRecorder(crawler)+ engine = make_engine(crawler)+ assert_state(engine, _EngineState.CREATED)+ with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):+ await engine.open_spider_async(close_if_idle=False)+ assert_state(engine, _EngineState.SPIDER_OPEN)+ assert not engine.running+ response = await engine.download_async(Request("data:,"))+ assert response.status == 200+ assert_state(engine, _EngineState.SPIDER_OPEN)+ await engine.close_async()+ assert_state(engine, _EngineState.STOPPED)+ assert_no_invalid_transition(caplog)+ assert recorder.names == ["spider_opened", "spider_closed"]+++@coroutine_test+async def test_close_created() -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)+ engine.downloader.close = Mock(wraps=engine.downloader.close) # type: ignore[method-assign]+ await engine.close_async()+ assert_state(engine, _EngineState.STOPPED)+ engine.downloader.close.assert_called()+++@coroutine_test+async def test_close_stopped(caplog: pytest.LogCaptureFixture) -> None:+ crawler = get_crawler(DefaultSpider)+ await crawler.crawl_async()+ engine = crawler.engine+ assert engine is not None+ assert_state(engine, _EngineState.STOPPED)+ with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):+ await engine.close_async()+ assert_state(engine, _EngineState.STOPPED)+ assert_no_invalid_transition(caplog)+++@coroutine_test+async def test_running_setter_deprecated() -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Setting ExecutionEngine.running is deprecated",+ ):+ engine.running = True+ assert engine.running is False # setting it has no effect+ engine.downloader.close()