Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support CloseSpider during spider startup (#7905)

Adrianc285f4cb18c3
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 35891ce8e..a6aac78ac 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -47,6 +47,13 @@ Additionally, they may also implement the following methods:
This method is called when the spider is opened.
+ .. versionchanged:: VERSION
+ Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+ It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before
+ it starts crawling, e.g. if a resource that the pipeline needs is
+ unavailable.
+
.. method:: close_spider(self)
This method is called when the spider is closed, before the
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index acbf7ddef..0a85c3c05 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -290,6 +290,13 @@ spider_opened
reserve per-spider resources, but can be used for any task that needs to be
performed when a spider is opened.
+ .. versionchanged:: VERSION
+ Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+ You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the
+ spider before it starts crawling, e.g. if a resource that the spider needs
+ is unavailable.
+
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
:param spider: the spider which has been opened
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b63d136a4..104daf399 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -248,7 +248,7 @@ class ExecutionEngine:
)
return deferred_from_coro(self.close_async())
- async def close_async(self) -> None:
+ async def close_async(self, *, reason: str = "shutdown") -> None:
"""
Gracefully close the execution engine.
If it has already been started, stop it. In all cases, close the spider and the downloader.
@@ -256,9 +256,7 @@ class ExecutionEngine:
if self.running:
await self.stop_async() # will also close spider and downloader
elif self.spider is not None:
- await self.close_spider_async(
- reason="shutdown"
- ) # will also close downloader
+ await self.close_spider_async(reason=reason) # will also close downloader
elif hasattr(self, "downloader"):
self.downloader.close()
@@ -557,10 +555,20 @@ class ExecutionEngine:
nextcall = CallLaterOnce(self._start_scheduled_requests)
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
self._slot = _Slot(close_if_idle, nextcall, scheduler)
- self._start = await self.scraper.spidermw.process_start()
- if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
- await maybe_deferred_to_future(d)
- await self.scraper.open_spider_async()
+ # A component that fails to start can ask for the spider to be closed.
+ # The rest of the startup runs anyway, so that components that are
+ # started also get stopped, and the request is honored once the spider
+ # is open.
+ close_spider_exc: CloseSpider | None = None
+ try:
+ self._start = await self.scraper.spidermw.process_start()
+ if hasattr(scheduler, "open") and (
+ d := scheduler.open(self.crawler.spider)
+ ):
+ await maybe_deferred_to_future(d)
+ await self.scraper.open_spider_async()
+ except CloseSpider as exc:
+ close_spider_exc = exc
stats = self.crawler.stats
if argument_is_required(stats.open_spider, "spider"):
warnings.warn(
@@ -572,9 +580,14 @@ class ExecutionEngine:
stats.open_spider(spider=self.crawler.spider)
else:
stats.open_spider()
- await self.signals.send_catch_log_async(
- signals.spider_opened, spider=self.crawler.spider
+ results = await self.signals.send_catch_log_async(
+ signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider
)
+ for _, result in results:
+ if isinstance(result, CloseSpider):
+ close_spider_exc = close_spider_exc or result
+ if close_spider_exc is not None:
+ raise close_spider_exc
def _spider_idle(self) -> None:
"""
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 44c4ffdcf..f0cfba6b9 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from scrapy import Spider
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
@@ -270,8 +270,12 @@ class Crawler:
self._apply_settings()
self._update_root_log_handler()
self.engine = self._create_engine()
- yield deferred_from_coro(self.engine.open_spider_async())
- yield deferred_from_coro(self.engine.start_async())
+ try:
+ yield deferred_from_coro(self.engine.open_spider_async())
+ except CloseSpider as exc:
+ yield deferred_from_coro(self.engine.close_async(reason=exc.reason))
+ else:
+ yield deferred_from_coro(self.engine.start_async())
except Exception:
self.crawling = False
if self._engine is not None:
@@ -300,8 +304,12 @@ class Crawler:
self._apply_settings()
self._update_root_log_handler()
self.engine = self._create_engine()
- await self.engine.open_spider_async()
- await self.engine.start_async()
+ try:
+ await self.engine.open_spider_async()
+ except CloseSpider as exc:
+ await self.engine.close_async(reason=exc.reason)
+ else:
+ await self.engine.start_async()
except Exception:
self.crawling = False
if self._engine is not None:
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index fa5bc1c1e..cd2560df1 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -56,12 +56,11 @@ class DontCloseSpider(Exception):
class CloseSpider(Exception):
- """Raised from a :ref:`spider callback <topics-spiders>` or from
- :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped.
+ """Raised from a :ref:`spider callback <topics-spiders>`, or while the
+ spider is starting, to request the spider to be closed/stopped.
.. versionchanged:: VERSION
- Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead
- of being reported as a start error.
+ Added support for raising it while the spider is starting.
*reason* is a string with the reason for closing.
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 8a3bceccb..53eb4a1f6 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -29,7 +29,9 @@ from tests.utils.engine import (
)
if TYPE_CHECKING:
- from collections.abc import AsyncIterator
+ from collections.abc import AsyncIterator, Generator
+
+ from twisted.internet.defer import Deferred
from tests.mockserver.http import MockServer
@@ -230,3 +232,51 @@ async def test_request_scheduled_signal():
f"{scheduler.enqueued!r} != [{keep_request!r}]"
)
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
+
+
+class ClosingPipeline:
+ def open_spider(self):
+ raise CloseSpider("pipeline_reason")
+
+
+class TestCloseSpiderOnStartup:
+ @coroutine_test
+ async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None:
+ closed: list[str] = []
+
+ def spider_closed(reason: str) -> None:
+ closed.append(reason)
+
+ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+ crawler.signals.connect(spider_closed, signals.spider_closed)
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
+ assert closed == ["pipeline_reason"]
+ assert "Traceback" not in caplog.text
+
+ @coroutine_test
+ async def test_spider_opened(self) -> None:
+ def spider_opened(spider: Spider) -> None:
+ raise CloseSpider("signal_reason")
+
+ crawler = get_crawler(DefaultSpider)
+ crawler.signals.connect(spider_opened, signals.spider_opened)
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("finish_reason") == "signal_reason"
+
+ @coroutine_test
+ async def test_startup_wins_over_spider_opened(self) -> None:
+ def spider_opened(spider: Spider) -> None:
+ raise CloseSpider("signal_reason")
+
+ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+ crawler.signals.connect(spider_opened, signals.spider_opened)
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
+
+ @inline_callbacks_test
+ def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]:
+ crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+ yield crawler.crawl()
+ assert crawler.stats.get_value("finish_reason") == "pipeline_reason"