Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refuse download_async() and crawl() during engine close. (#8319)

Andrey Rakhmatullind23e63e5215e
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 848e23348..254652c1e 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -167,6 +167,7 @@ class _Slot:
def _maybe_fire_closing(self) -> None:
if self.closing is not None and not self.inprogress:
+ assert not self.closing.called
self.nextcall.cancel()
if self.heartbeat.running:
self.heartbeat.stop()
@@ -591,10 +592,23 @@ class ExecutionEngine:
return False
return not self._slot.scheduler.has_pending_requests()
+ def _drained(self) -> bool:
+ """Return whether the spider close has stopped waiting for the
+ requests in progress.
+
+ The close then closes the downloader and the scheduler, so new
+ requests can be neither downloaded nor scheduled.
+ """
+ return self._slot is None or (
+ self._slot.closing is not None and self._slot.closing.called
+ )
+
def crawl(self, request: Request) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
+ if self._drained():
+ raise RuntimeError(f"Cannot crawl {request}: the spider is closing")
self._schedule_request(request)
self._slot.nextcall.schedule() # type: ignore[union-attr]
@@ -649,6 +663,9 @@ class ExecutionEngine:
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
while True:
+ # This also refuses the requests that replace a downloaded request.
+ if self._drained():
+ raise RuntimeError(f"Cannot download {request}: the spider is closing")
assert self._slot is not None
self._slot.add_request(request)
try:
diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py
index d89adeeee..f23719f26 100644
--- a/tests/test_engine_download.py
+++ b/tests/test_engine_download.py
@@ -21,6 +21,7 @@ def engine() -> ExecutionEngine:
engine.downloader.close()
engine.downloader = Mock()
engine._slot = Mock()
+ engine._slot.closing = None
engine._slot.inprogress = set()
return engine
diff --git a/tests/test_engine_lifecycle.py b/tests/test_engine_lifecycle.py
index 456e36f9b..50d0abfe3 100644
--- a/tests/test_engine_lifecycle.py
+++ b/tests/test_engine_lifecycle.py
@@ -7,10 +7,11 @@ from unittest.mock import Mock
import pytest
from twisted.internet.defer import Deferred
-from scrapy import Request, signals
+from scrapy import Request, Spider, signals
from scrapy.core.engine import EngineState, ExecutionEngine
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
+from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@@ -21,9 +22,10 @@ from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import AsyncIterator, Callable
from scrapy.crawler import Crawler
+ from scrapy.http import Response
from scrapy.utils._stopmode import _StopMode
@@ -637,6 +639,165 @@ class TestDoubleClose:
assert recorder.names == NORMAL_SIGNAL_ORDER
+class TestRequestsDuringClose:
+ """Requests sent while the spider is closing are processed as long as the
+ close waits for them, and refused afterwards."""
+
+ @coroutine_test
+ async def test_from_callback(self) -> None:
+ """Callbacks of the responses in progress when a graceful stop starts
+ can still download requests and schedule them."""
+ reached: Deferred[None] = Deferred()
+ proceed: Deferred[None] = Deferred()
+ bodies: list[bytes] = []
+
+ class CallbackSpider(Spider):
+ name = "callback"
+
+ async def start(self) -> AsyncIterator[Request]:
+ yield Request("data:,a")
+
+ async def parse(self, response: Response) -> None:
+ reached.callback(None)
+ await maybe_deferred_to_future(proceed)
+ assert self.crawler.engine is not None
+ response = await self.crawler.engine.download_async(Request("data:,b"))
+ bodies.append(response.body)
+ self.crawler.engine.crawl(Request("data:,c"))
+
+ crawler = get_crawler(CallbackSpider)
+ crawl_dfd = deferred_from_coro(crawler.crawl_async())
+ await maybe_deferred_to_future(reached)
+ stop_dfd = deferred_from_coro(crawler.stop_async())
+ await sleep(0)
+ assert crawler.engine is not None
+ assert_state(crawler.engine, EngineState.SPIDER_CLOSING)
+ proceed.callback(None)
+ await maybe_deferred_to_future(stop_dfd)
+ await maybe_deferred_to_future(crawl_dfd)
+ assert bodies == [b"b"]
+ assert crawler.stats.get_value("scheduler/enqueued") == 2
+ assert crawler.stats.get_value("finish_reason") == "shutdown"
+
+ @coroutine_test
+ async def test_replacement_request(self, caplog: pytest.LogCaptureFixture) -> None:
+ """When a downloader middleware replaces a request downloaded with
+ download_async() after the close has stopped waiting for the requests
+ in progress, the replacement is not downloaded."""
+ reached: Deferred[None] = Deferred()
+ proceed: Deferred[None] = Deferred()
+ errors: list[Exception] = []
+
+ class ReplacingMiddleware:
+ async def process_request(self, request: Request) -> None:
+ if request.url == "data:,1":
+ reached.callback(None)
+ await maybe_deferred_to_future(proceed)
+
+ def process_response(
+ self, request: Request, response: Response
+ ) -> Request | Response:
+ if request.url == "data:,1":
+ return Request("data:,2")
+ return response
+
+ async def download() -> None:
+ assert crawler.engine is not None
+ try:
+ await crawler.engine.download_async(Request("data:,1"))
+ except Exception as e:
+ errors.append(e)
+
+ def engine_started() -> None:
+ _schedule_coro(download())
+
+ crawler = get_crawler(
+ DefaultSpider,
+ {"DOWNLOADER_MIDDLEWARES": {ReplacingMiddleware: 0}},
+ )
+ crawler.signals.connect(engine_started, signals.engine_started)
+ crawl_dfd = deferred_from_coro(crawler.crawl_async())
+ await maybe_deferred_to_future(reached)
+ stop_dfd = deferred_from_coro(crawler.stop_async())
+ await sleep(0)
+ assert crawler.engine is not None
+ assert_state(crawler.engine, EngineState.SPIDER_CLOSING)
+ proceed.callback(None)
+ await maybe_deferred_to_future(stop_dfd)
+ await maybe_deferred_to_future(crawl_dfd)
+ assert len(errors) == 1
+ assert isinstance(errors[0], RuntimeError)
+ assert "Cannot download <GET data:,2>: the spider is closing" in str(errors[0])
+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
+ assert crawler.stats.get_value("downloader/request_count") == 1
+ assert crawler.stats.get_value("finish_reason") == "shutdown"
+
+ @pytest.mark.parametrize("source", ["pipeline", "spider_closed"])
+ @coroutine_test
+ async def test_download_after_drain(
+ self, source: str, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Pipelines' close_spider() and spider_closed handlers run after the
+ downloads have finished and cannot start new ones."""
+ errors: list[Exception] = []
+
+ async def download() -> None:
+ assert crawler.engine is not None
+ try:
+ await crawler.engine.download_async(Request("data:,"))
+ except Exception as e:
+ errors.append(e)
+
+ class DownloadingPipeline:
+ async def close_spider(self) -> None:
+ await download()
+
+ if source == "pipeline":
+ crawler = get_crawler(
+ DefaultSpider, {"ITEM_PIPELINES": {DownloadingPipeline: 0}}
+ )
+ else:
+ crawler = get_crawler(DefaultSpider)
+ crawler.signals.connect(download, signals.spider_closed)
+ await crawler.crawl_async()
+ assert len(errors) == 1
+ assert isinstance(errors[0], RuntimeError)
+ assert "spider is closing" in str(errors[0])
+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
+ assert crawler.stats.get_value("finish_reason") == "finished"
+
+ @pytest.mark.parametrize("source", ["pipeline", "spider_closed"])
+ @coroutine_test
+ async def test_crawl_after_drain(self, source: str) -> None:
+ """Pipelines' close_spider() and spider_closed handlers run after the
+ downloads have finished and cannot schedule new ones."""
+ errors: list[Exception] = []
+
+ def crawl() -> None:
+ assert crawler.engine is not None
+ try:
+ crawler.engine.crawl(Request("data:,"))
+ except Exception as e:
+ errors.append(e)
+
+ class CrawlingPipeline:
+ def close_spider(self) -> None:
+ crawl()
+
+ if source == "pipeline":
+ crawler = get_crawler(
+ DefaultSpider, {"ITEM_PIPELINES": {CrawlingPipeline: 0}}
+ )
+ else:
+ crawler = get_crawler(DefaultSpider)
+ crawler.signals.connect(crawl, signals.spider_closed)
+ await crawler.crawl_async()
+ assert len(errors) == 1
+ assert isinstance(errors[0], RuntimeError)
+ assert "spider is closing" in str(errors[0])
+ assert crawler.stats.get_value("scheduler/enqueued") is None
+
+
class TestInvalidTransitions:
"""Calls that the state does not allow raise instead of transitioning, and
a transition that the transition table does not allow is logged."""