Scrapy, a fast high-level web crawling & scraping framework for Python.
Support CloseSpider during spider startup (#7905)
Adrianc285f4cb18c3
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rstindex 35891ce8e..a6aac78ac 100644--- a/docs/topics/item-pipeline.rst+++ b/docs/topics/item-pipeline.rst@@ -47,6 +47,13 @@ Additionally, they may also implement the following methods:This method is called when the spider is opened.+ .. versionchanged:: VERSION+ Added support for :exc:`~scrapy.exceptions.CloseSpider`.++ It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before+ it starts crawling, e.g. if a resource that the pipeline needs is+ unavailable.+.. method:: close_spider(self)This method is called when the spider is closed, before thediff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex acbf7ddef..0a85c3c05 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -290,6 +290,13 @@ spider_openedreserve per-spider resources, but can be used for any task that needs to beperformed when a spider is opened.+ .. versionchanged:: VERSION+ Added support for :exc:`~scrapy.exceptions.CloseSpider`.++ You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the+ spider before it starts crawling, e.g. if a resource that the spider needs+ is unavailable.+This signal supports :ref:`asynchronous handlers <signal-deferred>`.:param spider: the spider which has been openeddiff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex b63d136a4..104daf399 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -248,7 +248,7 @@ class ExecutionEngine:)return deferred_from_coro(self.close_async())- async def close_async(self) -> None:+ async def close_async(self, *, reason: str = "shutdown") -> None:"""Gracefully close the execution engine.If it has already been started, stop it. In all cases, close the spider and the downloader.@@ -256,9 +256,7 @@ class ExecutionEngine:if self.running:await self.stop_async() # will also close spider and downloaderelif self.spider is not None:- await self.close_spider_async(- reason="shutdown"- ) # will also close downloader+ await self.close_spider_async(reason=reason) # will also close downloaderelif hasattr(self, "downloader"):self.downloader.close()@@ -557,10 +555,20 @@ class ExecutionEngine:nextcall = CallLaterOnce(self._start_scheduled_requests)scheduler = build_from_crawler(self.scheduler_cls, self.crawler)self._slot = _Slot(close_if_idle, nextcall, scheduler)- self._start = await self.scraper.spidermw.process_start()- if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):- await maybe_deferred_to_future(d)- await self.scraper.open_spider_async()+ # A component that fails to start can ask for the spider to be closed.+ # The rest of the startup runs anyway, so that components that are+ # started also get stopped, and the request is honored once the spider+ # is open.+ close_spider_exc: CloseSpider | None = None+ try:+ self._start = await self.scraper.spidermw.process_start()+ if hasattr(scheduler, "open") and (+ d := scheduler.open(self.crawler.spider)+ ):+ await maybe_deferred_to_future(d)+ await self.scraper.open_spider_async()+ except CloseSpider as exc:+ close_spider_exc = excstats = self.crawler.statsif argument_is_required(stats.open_spider, "spider"):warnings.warn(@@ -572,9 +580,14 @@ class ExecutionEngine:stats.open_spider(spider=self.crawler.spider)else:stats.open_spider()- await self.signals.send_catch_log_async(- signals.spider_opened, spider=self.crawler.spider+ results = await self.signals.send_catch_log_async(+ signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider)+ for _, result in results:+ if isinstance(result, CloseSpider):+ close_spider_exc = close_spider_exc or result+ if close_spider_exc is not None:+ raise close_spider_excdef _spider_idle(self) -> None:"""diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 44c4ffdcf..f0cfba6b9 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom scrapy import Spiderfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEngine-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManagerfrom scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager@@ -270,8 +270,12 @@ class Crawler:self._apply_settings()self._update_root_log_handler()self.engine = self._create_engine()- yield deferred_from_coro(self.engine.open_spider_async())- yield deferred_from_coro(self.engine.start_async())+ try:+ yield deferred_from_coro(self.engine.open_spider_async())+ except CloseSpider as exc:+ yield deferred_from_coro(self.engine.close_async(reason=exc.reason))+ else:+ yield deferred_from_coro(self.engine.start_async())except Exception:self.crawling = Falseif self._engine is not None:@@ -300,8 +304,12 @@ class Crawler:self._apply_settings()self._update_root_log_handler()self.engine = self._create_engine()- await self.engine.open_spider_async()- await self.engine.start_async()+ try:+ await self.engine.open_spider_async()+ except CloseSpider as exc:+ await self.engine.close_async(reason=exc.reason)+ else:+ await self.engine.start_async()except Exception:self.crawling = Falseif self._engine is not None:diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex fa5bc1c1e..cd2560df1 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -56,12 +56,11 @@ class DontCloseSpider(Exception):class CloseSpider(Exception):- """Raised from a :ref:`spider callback <topics-spiders>` or from- :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped.+ """Raised from a :ref:`spider callback <topics-spiders>`, or while the+ spider is starting, to request the spider to be closed/stopped... versionchanged:: VERSION- Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead- of being reported as a start error.+ Added support for raising it while the spider is starting.*reason* is a string with the reason for closing.diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 8a3bceccb..53eb4a1f6 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -29,7 +29,9 @@ from tests.utils.engine import ()if TYPE_CHECKING:- from collections.abc import AsyncIterator+ from collections.abc import AsyncIterator, Generator++ from twisted.internet.defer import Deferredfrom tests.mockserver.http import MockServer@@ -230,3 +232,51 @@ async def test_request_scheduled_signal():f"{scheduler.enqueued!r} != [{keep_request!r}]")crawler.signals.disconnect(signal_handler, signals.request_scheduled)+++class ClosingPipeline:+ def open_spider(self):+ raise CloseSpider("pipeline_reason")+++class TestCloseSpiderOnStartup:+ @coroutine_test+ async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None:+ closed: list[str] = []++ def spider_closed(reason: str) -> None:+ closed.append(reason)++ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})+ crawler.signals.connect(spider_closed, signals.spider_closed)+ with caplog.at_level(logging.INFO):+ await crawler.crawl_async()+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"+ assert closed == ["pipeline_reason"]+ assert "Traceback" not in caplog.text++ @coroutine_test+ async def test_spider_opened(self) -> None:+ def spider_opened(spider: Spider) -> None:+ raise CloseSpider("signal_reason")++ crawler = get_crawler(DefaultSpider)+ crawler.signals.connect(spider_opened, signals.spider_opened)+ await crawler.crawl_async()+ assert crawler.stats.get_value("finish_reason") == "signal_reason"++ @coroutine_test+ async def test_startup_wins_over_spider_opened(self) -> None:+ def spider_opened(spider: Spider) -> None:+ raise CloseSpider("signal_reason")++ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})+ crawler.signals.connect(spider_opened, signals.spider_opened)+ await crawler.crawl_async()+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"++ @inline_callbacks_test+ def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]:+ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})+ yield crawler.crawl()+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"