Scrapy, a fast high-level web crawling & scraping framework for Python.
Refuse download_async() and crawl() during engine close. (#8319)
Andrey Rakhmatullind23e63e5215e
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 848e23348..254652c1e 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -167,6 +167,7 @@ class _Slot:def _maybe_fire_closing(self) -> None:if self.closing is not None and not self.inprogress:+ assert not self.closing.calledself.nextcall.cancel()if self.heartbeat.running:self.heartbeat.stop()@@ -591,10 +592,23 @@ class ExecutionEngine:return Falsereturn not self._slot.scheduler.has_pending_requests()+ def _drained(self) -> bool:+ """Return whether the spider close has stopped waiting for the+ requests in progress.++ The close then closes the downloader and the scheduler, so new+ requests can be neither downloaded nor scheduled.+ """+ return self._slot is None or (+ self._slot.closing is not None and self._slot.closing.called+ )+def crawl(self, request: Request) -> None:"""Inject the request into the spider <-> downloader pipeline"""if self.spider is None:raise RuntimeError(f"No open spider to crawl: {request}")+ if self._drained():+ raise RuntimeError(f"Cannot crawl {request}: the spider is closing")self._schedule_request(request)self._slot.nextcall.schedule() # type: ignore[union-attr]@@ -649,6 +663,9 @@ class ExecutionEngine:if self.spider is None:raise RuntimeError(f"No open spider to crawl: {request}")while True:+ # This also refuses the requests that replace a downloaded request.+ if self._drained():+ raise RuntimeError(f"Cannot download {request}: the spider is closing")assert self._slot is not Noneself._slot.add_request(request)try:diff --git a/tests/test_engine_download.py b/tests/test_engine_download.pyindex d89adeeee..f23719f26 100644--- a/tests/test_engine_download.py+++ b/tests/test_engine_download.py@@ -21,6 +21,7 @@ def engine() -> ExecutionEngine:engine.downloader.close()engine.downloader = Mock()engine._slot = Mock()+ engine._slot.closing = Noneengine._slot.inprogress = set()return enginediff --git a/tests/test_engine_lifecycle.py b/tests/test_engine_lifecycle.pyindex 456e36f9b..50d0abfe3 100644--- a/tests/test_engine_lifecycle.py+++ b/tests/test_engine_lifecycle.py@@ -7,10 +7,11 @@ from unittest.mock import Mockimport pytestfrom twisted.internet.defer import Deferred-from scrapy import Request, signals+from scrapy import Request, Spider, signalsfrom scrapy.core.engine import EngineState, ExecutionEnginefrom scrapy.core.scheduler import BaseSchedulerfrom scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning+from scrapy.utils.asyncio import sleepfrom scrapy.utils.defer import (_schedule_coro,deferred_from_coro,@@ -21,9 +22,10 @@ from scrapy.utils.test import get_crawlerfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import AsyncIterator, Callablefrom scrapy.crawler import Crawler+ from scrapy.http import Responsefrom scrapy.utils._stopmode import _StopMode@@ -637,6 +639,165 @@ class TestDoubleClose:assert recorder.names == NORMAL_SIGNAL_ORDER+class TestRequestsDuringClose:+ """Requests sent while the spider is closing are processed as long as the+ close waits for them, and refused afterwards."""++ @coroutine_test+ async def test_from_callback(self) -> None:+ """Callbacks of the responses in progress when a graceful stop starts+ can still download requests and schedule them."""+ reached: Deferred[None] = Deferred()+ proceed: Deferred[None] = Deferred()+ bodies: list[bytes] = []++ class CallbackSpider(Spider):+ name = "callback"++ async def start(self) -> AsyncIterator[Request]:+ yield Request("data:,a")++ async def parse(self, response: Response) -> None:+ reached.callback(None)+ await maybe_deferred_to_future(proceed)+ assert self.crawler.engine is not None+ response = await self.crawler.engine.download_async(Request("data:,b"))+ bodies.append(response.body)+ self.crawler.engine.crawl(Request("data:,c"))++ crawler = get_crawler(CallbackSpider)+ crawl_dfd = deferred_from_coro(crawler.crawl_async())+ await maybe_deferred_to_future(reached)+ stop_dfd = deferred_from_coro(crawler.stop_async())+ await sleep(0)+ assert crawler.engine is not None+ assert_state(crawler.engine, EngineState.SPIDER_CLOSING)+ proceed.callback(None)+ await maybe_deferred_to_future(stop_dfd)+ await maybe_deferred_to_future(crawl_dfd)+ assert bodies == [b"b"]+ assert crawler.stats.get_value("scheduler/enqueued") == 2+ assert crawler.stats.get_value("finish_reason") == "shutdown"++ @coroutine_test+ async def test_replacement_request(self, caplog: pytest.LogCaptureFixture) -> None:+ """When a downloader middleware replaces a request downloaded with+ download_async() after the close has stopped waiting for the requests+ in progress, the replacement is not downloaded."""+ reached: Deferred[None] = Deferred()+ proceed: Deferred[None] = Deferred()+ errors: list[Exception] = []++ class ReplacingMiddleware:+ async def process_request(self, request: Request) -> None:+ if request.url == "data:,1":+ reached.callback(None)+ await maybe_deferred_to_future(proceed)++ def process_response(+ self, request: Request, response: Response+ ) -> Request | Response:+ if request.url == "data:,1":+ return Request("data:,2")+ return response++ async def download() -> None:+ assert crawler.engine is not None+ try:+ await crawler.engine.download_async(Request("data:,1"))+ except Exception as e:+ errors.append(e)++ def engine_started() -> None:+ _schedule_coro(download())++ crawler = get_crawler(+ DefaultSpider,+ {"DOWNLOADER_MIDDLEWARES": {ReplacingMiddleware: 0}},+ )+ crawler.signals.connect(engine_started, signals.engine_started)+ crawl_dfd = deferred_from_coro(crawler.crawl_async())+ await maybe_deferred_to_future(reached)+ stop_dfd = deferred_from_coro(crawler.stop_async())+ await sleep(0)+ assert crawler.engine is not None+ assert_state(crawler.engine, EngineState.SPIDER_CLOSING)+ proceed.callback(None)+ await maybe_deferred_to_future(stop_dfd)+ await maybe_deferred_to_future(crawl_dfd)+ assert len(errors) == 1+ assert isinstance(errors[0], RuntimeError)+ assert "Cannot download <GET data:,2>: the spider is closing" in str(errors[0])+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR]+ assert crawler.stats.get_value("downloader/request_count") == 1+ assert crawler.stats.get_value("finish_reason") == "shutdown"++ @pytest.mark.parametrize("source", ["pipeline", "spider_closed"])+ @coroutine_test+ async def test_download_after_drain(+ self, source: str, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Pipelines' close_spider() and spider_closed handlers run after the+ downloads have finished and cannot start new ones."""+ errors: list[Exception] = []++ async def download() -> None:+ assert crawler.engine is not None+ try:+ await crawler.engine.download_async(Request("data:,"))+ except Exception as e:+ errors.append(e)++ class DownloadingPipeline:+ async def close_spider(self) -> None:+ await download()++ if source == "pipeline":+ crawler = get_crawler(+ DefaultSpider, {"ITEM_PIPELINES": {DownloadingPipeline: 0}}+ )+ else:+ crawler = get_crawler(DefaultSpider)+ crawler.signals.connect(download, signals.spider_closed)+ await crawler.crawl_async()+ assert len(errors) == 1+ assert isinstance(errors[0], RuntimeError)+ assert "spider is closing" in str(errors[0])+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR]+ assert crawler.stats.get_value("finish_reason") == "finished"++ @pytest.mark.parametrize("source", ["pipeline", "spider_closed"])+ @coroutine_test+ async def test_crawl_after_drain(self, source: str) -> None:+ """Pipelines' close_spider() and spider_closed handlers run after the+ downloads have finished and cannot schedule new ones."""+ errors: list[Exception] = []++ def crawl() -> None:+ assert crawler.engine is not None+ try:+ crawler.engine.crawl(Request("data:,"))+ except Exception as e:+ errors.append(e)++ class CrawlingPipeline:+ def close_spider(self) -> None:+ crawl()++ if source == "pipeline":+ crawler = get_crawler(+ DefaultSpider, {"ITEM_PIPELINES": {CrawlingPipeline: 0}}+ )+ else:+ crawler = get_crawler(DefaultSpider)+ crawler.signals.connect(crawl, signals.spider_closed)+ await crawler.crawl_async()+ assert len(errors) == 1+ assert isinstance(errors[0], RuntimeError)+ assert "spider is closing" in str(errors[0])+ assert crawler.stats.get_value("scheduler/enqueued") is None++class TestInvalidTransitions:"""Calls that the state does not allow raise instead of transitioning, anda transition that the transition table does not allow is logged."""