Scrapy, a fast high-level web crawling & scraping framework for Python.
Further engine refactoring (#8272)
Andrey Rakhmatullinaef037d6fd57
diff --git a/docs/news.rst b/docs/news.rstindex b673ca788..c133a5f53 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -57,7 +57,12 @@ Backward-incompatible changescalled while the spider is being opened, now closes the spider as soonas it is open, instead of being ignored.- (TBD)+ - A :exc:`~scrapy.exceptions.CloseSpider` exception raised while the spider+ is opening now closes the spider as soon as it is open, instead of being+ re-raised by+ :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`.++ (:gh:`6916`, :gh:`8217`, :gh:`8272`)- Setting the :attr:`ExecutionEngine.running<scrapy.core.engine.ExecutionEngine.running>` attribute no longer has anydiff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex d45024bf3..03bf4d175 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -474,8 +474,9 @@ class ExecutionEngine:await self.stop_async()def _start_scheduled_requests(self) -> None:- if self._slot is None or self._slot.closing is not None or self.paused:+ if self._state is not EngineState.RUNNING or self.paused:return+ assert self._slot is not Nonewhile not self.needs_backout():if not self._start_scheduled_request():@@ -699,6 +700,17 @@ class ExecutionEngine:"""Open the spider.Raises :exc:`RuntimeError` if a spider has already been opened.++ If an exception is raised during opening, the spider is still+ considered open (the engine always transitions to the+ :attr:`EngineState.SPIDER_OPEN` state).++ .. versionchanged:: VERSION+ A :exc:`~scrapy.exceptions.CloseSpider` exception raised while the+ spider is opening now closes the spider instead of being re-raised.++ If the engine is requested to close the spider during opening, that is+ done right after the spider is opened."""assert self.crawler.spiderif self._state is EngineState.STOPPED:@@ -721,7 +733,6 @@ class ExecutionEngine:# The rest of the startup runs anyway, so that components that are# started also get stopped, and the request is honored once the spider# is open.- close_spider_exc: CloseSpider | None = Nonetry:self._start = await self.scraper.spidermw.process_start()if hasattr(scheduler, "open") and (@@ -730,7 +741,7 @@ class ExecutionEngine:await maybe_deferred_to_future(d)await self.scraper.open_spider_async()except CloseSpider as exc:- close_spider_exc = exc+ await self.close_spider_async(reason=exc.reason, error=exc.error)stats = self.crawler.statsif argument_is_required(stats.open_spider, "spider"):warnings.warn(@@ -747,14 +758,14 @@ class ExecutionEngine:)for _, result in results:if isinstance(result, CloseSpider):- close_spider_exc = close_spider_exc or result- if close_spider_exc is not None:- raise close_spider_exc+ await self.close_spider_async(+ reason=result.reason, error=result.error+ )finally:# Partially open counts as open because we still want to close the# components that were opened (close_spider_async() will do that).self._transition_to(EngineState.SPIDER_OPEN)- await self._close_spider_if_pending()+ await self._close_spider_if_pending()async def _close_spider_if_pending(self) -> None:"""Perform a close requested while the spider was opening."""diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex cbe91807d..66ac0e642 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom scrapy import Spiderfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import EngineState, ExecutionEngine-from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManagerfrom scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager@@ -302,17 +302,11 @@ class Crawler:self._apply_settings()self._update_logging()self.engine = ExecutionEngine(self)- try:- yield deferred_from_coro(self.engine.open_spider_async())- except CloseSpider as exc:- yield deferred_from_coro(- self.engine.close_async(reason=exc.reason, error=exc.error)- )- else:- # The spider may have been closed already, e.g. by an- # extension, in which case the engine is already stopped.- if self.engine.state is EngineState.SPIDER_OPEN:- yield deferred_from_coro(self.engine.start_async())+ yield deferred_from_coro(self.engine.open_spider_async())+ # The spider may have been closed already, e.g. by an+ # extension, in which case the engine is already stopped.+ if self.engine.state is EngineState.SPIDER_OPEN:+ yield deferred_from_coro(self.engine.start_async())except Exception:if self._engine is not None:yield deferred_from_coro(self._engine.close_async())@@ -342,15 +336,11 @@ class Crawler:self._apply_settings()self._update_logging()self.engine = ExecutionEngine(self)- try:- await self.engine.open_spider_async()- except CloseSpider as exc:- await self.engine.close_async(reason=exc.reason, error=exc.error)- else:- # The spider may have been closed already, e.g. by an- # extension, in which case the engine is already stopped.- if self.engine.state is EngineState.SPIDER_OPEN:- await self.engine.start_async()+ await self.engine.open_spider_async()+ # The spider may have been closed already, e.g. by an+ # extension, in which case the engine is already stopped.+ if self.engine.state is EngineState.SPIDER_OPEN:+ await self.engine.start_async()except Exception:if self._engine is not None:await self._engine.close_async()diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 6d4b06967..019badb59 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -19,6 +19,7 @@ from twisted.python import threadablefrom w3lib.url import any_to_uriimport scrapy+from scrapy.core.engine import EngineStatefrom scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningfrom scrapy.http import Request, Response@@ -190,6 +191,8 @@ class Shell:self.crawler.spider = spiderawait self.crawler.engine.open_spider_async(close_if_idle=False)+ if self.crawler.engine.state is not EngineState.SPIDER_OPEN:+ raise RuntimeError(f"Spider {spider.name!r} was closed while opening")self.spider = spiderdef fetch(diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 3550265e8..a483f5abd 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -13,6 +13,7 @@ import pytestfrom pexpect.popen_spawn import PopenSpawnfrom scrapy import Spider+from scrapy.core.engine import EngineStatefrom scrapy.http import Request, Responsefrom scrapy.shell import Shell, inspect_responsefrom scrapy.utils.reactor import _asyncio_reactor_path@@ -187,9 +188,22 @@ class TestShellCommandWithSpider(TestProjectBase):(proj_path / self.project_name / "spiders" / "myspider.py").write_text("""import scrapy+from scrapy.exceptions import CloseSpiderclass MySpider(scrapy.Spider):name = "myspider"++class ClosingSpider(scrapy.Spider):+ name = "closing"++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ crawler.signals.connect(spider.spider_opened, scrapy.signals.spider_opened)+ return spider++ def spider_opened(self):+ raise CloseSpider("unavailable")""",encoding="utf-8",)@@ -207,6 +221,22 @@ class MySpider(scrapy.Spider):assert ret == 0, errassert out.strip() == "myspider"+ def test_spider_closed_on_open(+ self, proj_path: Path, mockserver: MockServer+ ) -> None:+ ret, _, err = proc(+ "shell",+ "--spider",+ "closing",+ mockserver.url("/text"),+ "-c",+ "response",+ cwd=proj_path,+ )+ assert ret == 1+ assert "Closing spider (unavailable)" in err+ assert "RuntimeError: Spider 'closing' was closed while opening" in err+class TestInteractiveShell:# Starting an interactive shell involves an interpreter start-up, Scrapy@@ -402,6 +432,7 @@ class TestShell:crawler = get_crawler()crawler.engine = MagicMock()crawler.engine.open_spider_async = AsyncMock()+ crawler.engine.state = EngineState.SPIDER_OPENshell = Shell(crawler)spider = Spider.from_crawler(crawler, "test")await shell._open_spider(spider)diff --git a/tests/test_engine_lifecycle.py b/tests/test_engine_lifecycle.pyindex d0a614fec..456e36f9b 100644--- a/tests/test_engine_lifecycle.py+++ b/tests/test_engine_lifecycle.py@@ -10,7 +10,7 @@ from twisted.internet.defer import Deferredfrom scrapy import Request, signalsfrom scrapy.core.engine import EngineState, ExecutionEnginefrom scrapy.core.scheduler import BaseScheduler-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarningfrom scrapy.utils.defer import (_schedule_coro,deferred_from_coro,@@ -37,6 +37,7 @@ class SignalRecorder:self.crawler = crawlerself.calls: list[tuple[str, EngineState]] = []self.close_reasons: list[str] = []+ self.close_errors: list[bool] = []# Keep strong references to the handlers: signal connections are weak.self._handlers = [self._make_handler(name) for name in self.SIGNALS]for name, handler in zip(self.SIGNALS, self._handlers, strict=True):@@ -48,6 +49,7 @@ class SignalRecorder:self.calls.append((name, self.crawler.engine.state))if name == "spider_closed":self.close_reasons.append(kwargs["reason"])+ self.close_errors.append(kwargs["error"])return handler@@ -82,6 +84,11 @@ class BlockingScheduler(BaseScheduler):return self.unblock_close+class ClosingPipeline:+ def open_spider(self) -> None:+ raise CloseSpider("early", error=True)++def make_engine(crawler: Crawler) -> ExecutionEngine:"""Create the engine of *crawler* as Crawler.crawl_async() (and scrapyshell) would, without opening or starting anything."""@@ -404,6 +411,89 @@ class TestCloseDuringOpen:assert_state(engine, EngineState.STOPPED)assert errors == [True]+ @coroutine_test+ async def test_close_spider_exception_from_component(self) -> None:+ """CloseSpider raised by a component while the spider is opening+ closes the spider once it is open, instead of being raised."""+ crawler = get_crawler(+ DefaultSpider, settings_dict={"ITEM_PIPELINES": {ClosingPipeline: 1}}+ )+ recorder = SignalRecorder(crawler)+ engine = make_engine(crawler)+ await engine.open_spider_async()+ assert_state(engine, EngineState.STOPPED)+ assert recorder.names == NEVER_STARTED_SIGNAL_ORDER+ assert recorder.close_reasons == ["early"]+ assert recorder.close_errors == [True]++ @coroutine_test+ async def test_close_spider_exception_from_spider_opened(self) -> None:+ """CloseSpider raised by a spider_opened handler closes the spider+ once it is open, instead of being raised."""+ crawler = get_crawler(DefaultSpider)+ recorder = SignalRecorder(crawler)+ engine = make_engine(crawler)++ def close(**kwargs: Any) -> None:+ raise CloseSpider("early", error=True)++ crawler.signals.connect(close, signals.spider_opened)+ await engine.open_spider_async()+ assert_state(engine, EngineState.STOPPED)+ assert recorder.names == NEVER_STARTED_SIGNAL_ORDER+ assert recorder.close_reasons == ["early"]+ assert recorder.close_errors == [True]++ @coroutine_test+ async def test_close_spider_call_wins_over_exception(self) -> None:+ """The first close requested while the spider is opening wins, and a+ CloseSpider raised by a spider_opened handler only counts as requested+ once all spider_opened handlers have finished."""+ crawler = get_crawler(DefaultSpider)+ recorder = SignalRecorder(crawler)+ engine = make_engine(crawler)++ def raise_close_spider(**kwargs: Any) -> None:+ raise CloseSpider("exception")++ async def call_close_spider(**kwargs: Any) -> None:+ await engine.close_spider_async(reason="call")++ crawler.signals.connect(raise_close_spider, signals.spider_opened)+ crawler.signals.connect(call_close_spider, signals.spider_opened)+ await engine.open_spider_async()+ assert_state(engine, EngineState.STOPPED)+ assert recorder.close_reasons == ["call"]++ @coroutine_test+ async def test_close_spider_during_failed_open(self) -> None:+ """A close requested while the spider is opening is performed even if+ opening the spider then fails."""+ opening: Deferred[None] = Deferred()+ fail_open: Deferred[None] = Deferred()++ class FailingPipeline:+ async def open_spider(self) -> None:+ opening.callback(None)+ await maybe_deferred_to_future(fail_open)+ raise ValueError("open failed")++ crawler = get_crawler(+ DefaultSpider, settings_dict={"ITEM_PIPELINES": {FailingPipeline: 1}}+ )+ recorder = SignalRecorder(crawler)+ crawl_dfd = deferred_from_coro(crawler.crawl_async())+ await maybe_deferred_to_future(opening)+ engine = crawler.engine+ assert engine is not None+ await engine.close_spider_async(reason="early", error=True)+ fail_open.callback(None)+ with pytest.raises(ValueError, match="open failed"):+ await maybe_deferred_to_future(crawl_dfd)+ assert_state(engine, EngineState.STOPPED)+ assert recorder.close_reasons == ["early"]+ assert recorder.close_errors == [True]+@coroutine_testasync def test_stop_during_open(self) -> None:"""stop_async() while the spider is opening is performed once the