Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Further engine refactoring (#8272)

Andrey Rakhmatullinaef037d6fd57
diff --git a/docs/news.rst b/docs/news.rst
index b673ca788..c133a5f53 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -57,7 +57,12 @@ Backward-incompatible changes
called while the spider is being opened, now closes the spider as soon
as it is open, instead of being ignored.
- (TBD)
+ - A :exc:`~scrapy.exceptions.CloseSpider` exception raised while the spider
+ is opening now closes the spider as soon as it is open, instead of being
+ re-raised by
+ :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`.
+
+ (:gh:`6916`, :gh:`8217`, :gh:`8272`)
- Setting the :attr:`ExecutionEngine.running
<scrapy.core.engine.ExecutionEngine.running>` attribute no longer has any
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index d45024bf3..03bf4d175 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -474,8 +474,9 @@ class ExecutionEngine:
await self.stop_async()
def _start_scheduled_requests(self) -> None:
- if self._slot is None or self._slot.closing is not None or self.paused:
+ if self._state is not EngineState.RUNNING or self.paused:
return
+ assert self._slot is not None
while not self.needs_backout():
if not self._start_scheduled_request():
@@ -699,6 +700,17 @@ class ExecutionEngine:
"""Open the spider.
Raises :exc:`RuntimeError` if a spider has already been opened.
+
+ If an exception is raised during opening, the spider is still
+ considered open (the engine always transitions to the
+ :attr:`EngineState.SPIDER_OPEN` state).
+
+ .. versionchanged:: VERSION
+ A :exc:`~scrapy.exceptions.CloseSpider` exception raised while the
+ spider is opening now closes the spider instead of being re-raised.
+
+ If the engine is requested to close the spider during opening, that is
+ done right after the spider is opened.
"""
assert self.crawler.spider
if self._state is EngineState.STOPPED:
@@ -721,7 +733,6 @@ class ExecutionEngine:
# The rest of the startup runs anyway, so that components that are
# started also get stopped, and the request is honored once the spider
# is open.
- close_spider_exc: CloseSpider | None = None
try:
self._start = await self.scraper.spidermw.process_start()
if hasattr(scheduler, "open") and (
@@ -730,7 +741,7 @@ class ExecutionEngine:
await maybe_deferred_to_future(d)
await self.scraper.open_spider_async()
except CloseSpider as exc:
- close_spider_exc = exc
+ await self.close_spider_async(reason=exc.reason, error=exc.error)
stats = self.crawler.stats
if argument_is_required(stats.open_spider, "spider"):
warnings.warn(
@@ -747,14 +758,14 @@ class ExecutionEngine:
)
for _, result in results:
if isinstance(result, CloseSpider):
- close_spider_exc = close_spider_exc or result
- if close_spider_exc is not None:
- raise close_spider_exc
+ await self.close_spider_async(
+ reason=result.reason, error=result.error
+ )
finally:
# Partially open counts as open because we still want to close the
# components that were opened (close_spider_async() will do that).
self._transition_to(EngineState.SPIDER_OPEN)
- await self._close_spider_if_pending()
+ await self._close_spider_if_pending()
async def _close_spider_if_pending(self) -> None:
"""Perform a close requested while the spider was opening."""
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index cbe91807d..66ac0e642 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from scrapy import Spider
from scrapy.addons import AddonManager
from scrapy.core.engine import EngineState, ExecutionEngine
-from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
@@ -302,17 +302,11 @@ class Crawler:
self._apply_settings()
self._update_logging()
self.engine = ExecutionEngine(self)
- try:
- yield deferred_from_coro(self.engine.open_spider_async())
- except CloseSpider as exc:
- yield deferred_from_coro(
- self.engine.close_async(reason=exc.reason, error=exc.error)
- )
- else:
- # The spider may have been closed already, e.g. by an
- # extension, in which case the engine is already stopped.
- if self.engine.state is EngineState.SPIDER_OPEN:
- yield deferred_from_coro(self.engine.start_async())
+ yield deferred_from_coro(self.engine.open_spider_async())
+ # The spider may have been closed already, e.g. by an
+ # extension, in which case the engine is already stopped.
+ if self.engine.state is EngineState.SPIDER_OPEN:
+ yield deferred_from_coro(self.engine.start_async())
except Exception:
if self._engine is not None:
yield deferred_from_coro(self._engine.close_async())
@@ -342,15 +336,11 @@ class Crawler:
self._apply_settings()
self._update_logging()
self.engine = ExecutionEngine(self)
- try:
- await self.engine.open_spider_async()
- except CloseSpider as exc:
- await self.engine.close_async(reason=exc.reason, error=exc.error)
- else:
- # The spider may have been closed already, e.g. by an
- # extension, in which case the engine is already stopped.
- if self.engine.state is EngineState.SPIDER_OPEN:
- await self.engine.start_async()
+ await self.engine.open_spider_async()
+ # The spider may have been closed already, e.g. by an
+ # extension, in which case the engine is already stopped.
+ if self.engine.state is EngineState.SPIDER_OPEN:
+ await self.engine.start_async()
except Exception:
if self._engine is not None:
await self._engine.close_async()
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 6d4b06967..019badb59 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -19,6 +19,7 @@ from twisted.python import threadable
from w3lib.url import any_to_uri
import scrapy
+from scrapy.core.engine import EngineState
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.http import Request, Response
@@ -190,6 +191,8 @@ class Shell:
self.crawler.spider = spider
await self.crawler.engine.open_spider_async(close_if_idle=False)
+ if self.crawler.engine.state is not EngineState.SPIDER_OPEN:
+ raise RuntimeError(f"Spider {spider.name!r} was closed while opening")
self.spider = spider
def fetch(
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 3550265e8..a483f5abd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -13,6 +13,7 @@ import pytest
from pexpect.popen_spawn import PopenSpawn
from scrapy import Spider
+from scrapy.core.engine import EngineState
from scrapy.http import Request, Response
from scrapy.shell import Shell, inspect_response
from scrapy.utils.reactor import _asyncio_reactor_path
@@ -187,9 +188,22 @@ class TestShellCommandWithSpider(TestProjectBase):
(proj_path / self.project_name / "spiders" / "myspider.py").write_text(
"""
import scrapy
+from scrapy.exceptions import CloseSpider
class MySpider(scrapy.Spider):
name = "myspider"
+
+class ClosingSpider(scrapy.Spider):
+ name = "closing"
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ crawler.signals.connect(spider.spider_opened, scrapy.signals.spider_opened)
+ return spider
+
+ def spider_opened(self):
+ raise CloseSpider("unavailable")
""",
encoding="utf-8",
)
@@ -207,6 +221,22 @@ class MySpider(scrapy.Spider):
assert ret == 0, err
assert out.strip() == "myspider"
+ def test_spider_closed_on_open(
+ self, proj_path: Path, mockserver: MockServer
+ ) -> None:
+ ret, _, err = proc(
+ "shell",
+ "--spider",
+ "closing",
+ mockserver.url("/text"),
+ "-c",
+ "response",
+ cwd=proj_path,
+ )
+ assert ret == 1
+ assert "Closing spider (unavailable)" in err
+ assert "RuntimeError: Spider 'closing' was closed while opening" in err
+
class TestInteractiveShell:
# Starting an interactive shell involves an interpreter start-up, Scrapy
@@ -402,6 +432,7 @@ class TestShell:
crawler = get_crawler()
crawler.engine = MagicMock()
crawler.engine.open_spider_async = AsyncMock()
+ crawler.engine.state = EngineState.SPIDER_OPEN
shell = Shell(crawler)
spider = Spider.from_crawler(crawler, "test")
await shell._open_spider(spider)
diff --git a/tests/test_engine_lifecycle.py b/tests/test_engine_lifecycle.py
index d0a614fec..456e36f9b 100644
--- a/tests/test_engine_lifecycle.py
+++ b/tests/test_engine_lifecycle.py
@@ -10,7 +10,7 @@ from twisted.internet.defer import Deferred
from scrapy import Request, signals
from scrapy.core.engine import EngineState, ExecutionEngine
from scrapy.core.scheduler import BaseScheduler
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@@ -37,6 +37,7 @@ class SignalRecorder:
self.crawler = crawler
self.calls: list[tuple[str, EngineState]] = []
self.close_reasons: list[str] = []
+ self.close_errors: list[bool] = []
# Keep strong references to the handlers: signal connections are weak.
self._handlers = [self._make_handler(name) for name in self.SIGNALS]
for name, handler in zip(self.SIGNALS, self._handlers, strict=True):
@@ -48,6 +49,7 @@ class SignalRecorder:
self.calls.append((name, self.crawler.engine.state))
if name == "spider_closed":
self.close_reasons.append(kwargs["reason"])
+ self.close_errors.append(kwargs["error"])
return handler
@@ -82,6 +84,11 @@ class BlockingScheduler(BaseScheduler):
return self.unblock_close
+class ClosingPipeline:
+ def open_spider(self) -> None:
+ raise CloseSpider("early", error=True)
+
+
def make_engine(crawler: Crawler) -> ExecutionEngine:
"""Create the engine of *crawler* as Crawler.crawl_async() (and scrapy
shell) would, without opening or starting anything."""
@@ -404,6 +411,89 @@ class TestCloseDuringOpen:
assert_state(engine, EngineState.STOPPED)
assert errors == [True]
+ @coroutine_test
+ async def test_close_spider_exception_from_component(self) -> None:
+ """CloseSpider raised by a component while the spider is opening
+ closes the spider once it is open, instead of being raised."""
+ crawler = get_crawler(
+ DefaultSpider, settings_dict={"ITEM_PIPELINES": {ClosingPipeline: 1}}
+ )
+ recorder = SignalRecorder(crawler)
+ engine = make_engine(crawler)
+ await engine.open_spider_async()
+ assert_state(engine, EngineState.STOPPED)
+ assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
+ assert recorder.close_reasons == ["early"]
+ assert recorder.close_errors == [True]
+
+ @coroutine_test
+ async def test_close_spider_exception_from_spider_opened(self) -> None:
+ """CloseSpider raised by a spider_opened handler closes the spider
+ once it is open, instead of being raised."""
+ crawler = get_crawler(DefaultSpider)
+ recorder = SignalRecorder(crawler)
+ engine = make_engine(crawler)
+
+ def close(**kwargs: Any) -> None:
+ raise CloseSpider("early", error=True)
+
+ crawler.signals.connect(close, signals.spider_opened)
+ await engine.open_spider_async()
+ assert_state(engine, EngineState.STOPPED)
+ assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
+ assert recorder.close_reasons == ["early"]
+ assert recorder.close_errors == [True]
+
+ @coroutine_test
+ async def test_close_spider_call_wins_over_exception(self) -> None:
+ """The first close requested while the spider is opening wins, and a
+ CloseSpider raised by a spider_opened handler only counts as requested
+ once all spider_opened handlers have finished."""
+ crawler = get_crawler(DefaultSpider)
+ recorder = SignalRecorder(crawler)
+ engine = make_engine(crawler)
+
+ def raise_close_spider(**kwargs: Any) -> None:
+ raise CloseSpider("exception")
+
+ async def call_close_spider(**kwargs: Any) -> None:
+ await engine.close_spider_async(reason="call")
+
+ crawler.signals.connect(raise_close_spider, signals.spider_opened)
+ crawler.signals.connect(call_close_spider, signals.spider_opened)
+ await engine.open_spider_async()
+ assert_state(engine, EngineState.STOPPED)
+ assert recorder.close_reasons == ["call"]
+
+ @coroutine_test
+ async def test_close_spider_during_failed_open(self) -> None:
+ """A close requested while the spider is opening is performed even if
+ opening the spider then fails."""
+ opening: Deferred[None] = Deferred()
+ fail_open: Deferred[None] = Deferred()
+
+ class FailingPipeline:
+ async def open_spider(self) -> None:
+ opening.callback(None)
+ await maybe_deferred_to_future(fail_open)
+ raise ValueError("open failed")
+
+ crawler = get_crawler(
+ DefaultSpider, settings_dict={"ITEM_PIPELINES": {FailingPipeline: 1}}
+ )
+ recorder = SignalRecorder(crawler)
+ crawl_dfd = deferred_from_coro(crawler.crawl_async())
+ await maybe_deferred_to_future(opening)
+ engine = crawler.engine
+ assert engine is not None
+ await engine.close_spider_async(reason="early", error=True)
+ fail_open.callback(None)
+ with pytest.raises(ValueError, match="open failed"):
+ await maybe_deferred_to_future(crawl_dfd)
+ assert_state(engine, EngineState.STOPPED)
+ assert recorder.close_reasons == ["early"]
+ assert recorder.close_errors == [True]
+
@coroutine_test
async def test_stop_during_open(self) -> None:
"""stop_async() while the spider is opening is performed once the