Scrapy, a fast high-level web crawling & scraping framework for Python.
Support fast crawler stops (#7455)
Adriana648c05dadad
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex 3bfbf8fb4..3edd7dbfd 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -19,7 +19,7 @@ jobs:checks:if: github.event.action != 'closed'runs-on: ubuntu-latest- timeout-minutes: 10+ timeout-minutes: 30env:# Make uv use the interpreter that actions/setup-python installed instead# of downloading one of its own.@@ -73,7 +73,7 @@ jobs:pre-commit:if: github.event.action != 'closed'runs-on: ubuntu-latest- timeout-minutes: 10+ timeout-minutes: 15steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.ymlindex 3e5015de9..e8a81f314 100644--- a/.github/workflows/tests-macos.yml+++ b/.github/workflows/tests-macos.yml@@ -20,7 +20,7 @@ jobs:if: github.event.action != 'closed'name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})runs-on: macos-latest- timeout-minutes: 15+ timeout-minutes: 60env:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}# Make uv use the interpreter that actions/setup-python installed insteaddiff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 62633f54f..595b81f48 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -20,7 +20,7 @@ jobs:if: github.event.action != 'closed'name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})runs-on: ubuntu-latest- timeout-minutes: 20+ timeout-minutes: 60env:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}# Make uv use the interpreter that actions/setup-python installed insteaddiff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex 45c9b4b35..70d8dba6b 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -20,7 +20,7 @@ jobs:if: github.event.action != 'closed'name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})runs-on: windows-latest- timeout-minutes: 20+ timeout-minutes: 60env:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}# Make uv use the interpreter that actions/setup-python installed insteaddiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 94672cae5..5969498aa 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -8,13 +8,13 @@ from datetime import datetimefrom time import monotonicfrom typing import TYPE_CHECKING, Any-from twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy import Request, Spider, signalsfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManager-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import DownloadCancelledError, ScrapyDeprecationWarningfrom scrapy.resolver import dnscachefrom scrapy.settings import SETTINGS_PRIORITIESfrom scrapy.utils.asyncio import (@@ -26,7 +26,6 @@ from scrapy.utils.asyncio import (from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (_process_pending_io,- _schedule_coro,deferred_from_coro,maybe_deferred_to_future,)@@ -202,6 +201,8 @@ class Downloader:DownloaderMiddlewareManager, crawler)self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None+ self._accepting_requests: bool = True+ self._download_tasks: dict[Request, Deferred[None]] = {}self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict("DOWNLOAD_SLOTS")@@ -266,6 +267,10 @@ class Downloader:# passed as download_func into self.middleware.download() in self.fetch()async def _enqueue_request(self, request: Request) -> Response:+ if not self._accepting_requests:+ raise DownloadCancelledError(+ "The downloader is shutting down and not accepting new requests"+ )key, slot = self._get_slot(request)request.meta[self.DOWNLOAD_SLOT] = keyslot.active.add(request)@@ -300,12 +305,21 @@ class Downloader:while slot.queue and slot.free_transfer_slots() > 0:slot.lastseen = nowrequest, queue_dfd = slot.queue.popleft()- _schedule_coro(self._wait_for_download(slot, request, queue_dfd))+ download_dfd = deferred_from_coro(+ self._wait_for_download(slot, request, queue_dfd)+ )+ assert isinstance(download_dfd, Deferred)+ self._download_tasks[request] = download_dfd+ download_dfd.addBoth(self._download_task_done, request)# prevent burst if inter-request delays were configuredif delay:self._process_queue(slot)break+ def _download_task_done(self, result: Any, request: Request) -> Any:+ self._download_tasks.pop(request, None)+ return result+def _latercall(self, slot: Slot) -> None:slot.latercall = Noneself._process_queue(slot)@@ -347,11 +361,55 @@ class Downloader:try:response = await self._download(slot, request)except Exception:- queue_dfd.errback(Failure())+ if not queue_dfd.called:+ queue_dfd.errback(Failure())else:queue_dfd.callback(response) # awaited in _enqueue_request()+ async def stop(self) -> int:+ self._accepting_requests = False++ dropped_count = 0++ for slot in self.slots.values():+ slot.close()+ while slot.queue:+ request, queue_dfd = slot.queue.popleft()+ dropped_count += 1+ if not queue_dfd.called:+ queue_dfd.errback(+ Failure(+ DownloadCancelledError(+ "Request dropped due to fast downloader shutdown"+ )+ )+ )+ self.signals.send_catch_log(+ signal=signals.request_left_downloader,+ request=request,+ spider=self.crawler.spider,+ )++ cancelled_dfds: list[Deferred[None]] = []+ for download_dfd in list(self._download_tasks.values()):+ if download_dfd.called:+ continue+ dropped_count += 1+ download_dfd.cancel()+ cancelled_dfds.append(download_dfd)++ if cancelled_dfds:+ # Wait for the cancellations themselves to settle, e.g. the+ # _download_tasks entries to be popped by _download_task_done(),+ # instead of guessing how many event loop iterations that takes.+ await maybe_deferred_to_future(+ DeferredList(cancelled_dfds, consumeErrors=True)+ )++ return dropped_count+def close(self) -> None:+ self._accepting_requests = Falseself._stop_slot_gc()for slot in self.slots.values():slot.close()diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex a03f362e6..c309b45f2 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -25,10 +25,12 @@ from scrapy.core.scraper import Scraperfrom scrapy.exceptions import (CloseSpider,DontCloseSpider,+ DownloadCancelledError,IgnoreRequest,ScrapyDeprecationWarning,)from scrapy.http import Request, Response+from scrapy.utils._stopmode import _max_stop_mode, _normalize_stop_mode, _StopModefrom scrapy.utils.asyncio import (AsyncioLoopingCall,create_looping_call,@@ -117,6 +119,8 @@ class ExecutionEngine:self.running: bool = Falseself._starting: bool = Falseself._stopping: bool = False+ self._stop_mode: _StopMode = "graceful"+ self._downloader_fast_stopped: bool = Falseself.paused: bool = Falseself._spider_closed_callback: Callable[[Spider], Coroutine[Any, Any, None] | Deferred[None] | None@@ -211,23 +215,36 @@ class ExecutionEngine:with contextlib.suppress(asyncio.exceptions.CancelledError):await maybe_deferred_to_future(self._closewait)- def stop(self) -> Deferred[None]: # pragma: no cover+ def stop(+ self, *, mode: _StopMode = "graceful"+ ) -> Deferred[None]: # pragma: no coverwarnings.warn("ExecutionEngine.stop() is deprecated, use stop_async() instead",ScrapyDeprecationWarning,stacklevel=2,)- return deferred_from_coro(self.stop_async())+ return deferred_from_coro(self.stop_async(mode=mode))- async def stop_async(self) -> None:+ async def stop_async(self, *, mode: _StopMode = "graceful") -> None:"""Gracefully stop the execution engine... versionadded:: 2.14"""- if not self._starting:+ mode = _normalize_stop_mode(mode, allow_force=False)++ if not self._starting and not self._stopping:raise RuntimeError("Engine not running")+ self._stop_mode = _max_stop_mode(self._stop_mode, mode)++ if self._stopping:+ if self.spider is not None and self._stop_mode == "fast":+ await self.close_spider_async(reason="shutdown", mode="fast")+ if self._closewait:+ await maybe_deferred_to_future(self._closewait)+ return+self.running = self._starting = Falseself._stopping = Trueif self._start_request_processing_awaitable is not None:@@ -242,7 +259,7 @@ class ExecutionEngine:self._start_request_processing_awaitable.cancel()self._start_request_processing_awaitable = Noneif self.spider is not None:- await self.close_spider_async(reason="shutdown")+ await self.close_spider_async(reason="shutdown", mode=self._stop_mode)await self.signals.send_catch_log_async(signal=signals.engine_stopped)if self._closewait:self._closewait.callback(None)@@ -420,6 +437,17 @@ class ExecutionEngine:def _handle_downloader_output(self, result: Request | Response | Failure, request: Request) -> Generator[Deferred[Any], Any, None]:+ if (+ isinstance(result, Failure)+ and self._stop_mode == "fast"+ and result.check(+ DownloadCancelledError,+ CancelledError,+ asyncio.exceptions.CancelledError,+ )+ ):+ return+# downloader middleware can return requests (for example, redirects)if isinstance(result, Request):self.crawl(result)@@ -616,20 +644,59 @@ class ExecutionEngine:_schedule_coro(self.close_spider_async(reason=ex.reason))def close_spider(- self, spider: Spider, reason: str = "cancelled"+ self,+ spider: Spider,+ reason: str = "cancelled",+ mode: _StopMode = "graceful",) -> Deferred[None]: # pragma: no coverwarnings.warn("ExecutionEngine.close_spider() is deprecated, use close_spider_async() instead",ScrapyDeprecationWarning,stacklevel=2,)- return deferred_from_coro(self.close_spider_async(reason=reason))+ return deferred_from_coro(self.close_spider_async(reason=reason, mode=mode))++ async def _fast_stop_downloader(self) -> None:+ if self._downloader_fast_stopped:+ return++ self._downloader_fast_stopped = True+ if not hasattr(self.downloader, "stop"):+ logger.warning(+ f"{type(self.downloader).__qualname__} does not implement "+ f"stop(), so pending downloads cannot be dropped and will be "+ f"finished before the spider closes",+ extra={"spider": self.spider},+ )+ return+ dropped_count = await self.downloader.stop()++ assert self.crawler.stats+ if dropped_count:+ self.crawler.stats.inc_value(+ "downloader/request_dropped_count", dropped_count+ )++ logger.info(+ "Fast shutdown dropped %(count)d downloader requests",+ {"count": dropped_count},+ extra={"spider": self.spider},+ )- async def close_spider_async(self, *, reason: str = "cancelled") -> None: # noqa: PLR0912+ # pylint: disable=too-many-statements+ async def close_spider_async( # noqa: PLR0912, PLR0915+ self,+ *,+ reason: str = "cancelled",+ mode: _StopMode = "graceful",+ ) -> None:"""Close (cancel) spider and clear all its outstanding requests... versionadded:: 2.14"""+ mode = _normalize_stop_mode(mode, allow_force=False)+ self._stop_mode = _max_stop_mode(self._stop_mode, mode)+if self.spider is None:raise RuntimeError("Spider not opened")@@ -637,6 +704,8 @@ class ExecutionEngine:raise RuntimeError("Engine slot not assigned")if self._slot.closing is not None:+ if self._stop_mode == "fast":+ await self._fast_stop_downloader()await maybe_deferred_to_future(self._slot.closing)return@@ -646,6 +715,9 @@ class ExecutionEngine:"Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider})+ if self._stop_mode == "fast":+ await self._fast_stop_downloader()+try:await self._slot.close()except Exception:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 5baa36907..b70f57435 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -24,7 +24,7 @@ from scrapy.http import Request, Responsefrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_availablefrom scrapy.utils.defer import (- _process_pending_io,+ _process_pending_io_before_callback,_schedule_coro,aiter_errback,deferred_from_coro,@@ -312,7 +312,7 @@ class Scraper:.. versionadded:: 2.13"""- await _process_pending_io()+ await _process_pending_io_before_callback()assert self.crawler.spiderif isinstance(result, Response):if getattr(result, "request", None) is None:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex dd1593118..0c2830c4c 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -20,7 +20,8 @@ from scrapy.extension import ExtensionManagerfrom scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader-from scrapy.utils.defer import deferred_from_coro+from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode+from scrapy.utils.defer import _DEFER_DELAY, deferred_from_coro, ensure_awaitablefrom scrapy.utils.log import (configure_logging,get_scrapy_root_handler,@@ -46,7 +47,7 @@ from scrapy.utils.reactorless import ()if TYPE_CHECKING:- from collections.abc import Awaitable, Generator, Iterable+ from collections.abc import Awaitable, Callable, Generator, Iterablefrom scrapy.logformatter import LogFormatterfrom scrapy.statscollectors import StatsCollector@@ -150,6 +151,9 @@ class Crawler:self._started: bool = Falseself.spider: Spider | None = None+ self._force_stop_callback: (+ Callable[[], Awaitable[None] | Deferred[None] | None] | None+ ) = Noneself._engine: ExecutionEngine | None = Noneself._extensions: ExtensionManager | None = None@@ -157,6 +161,12 @@ class Crawler:self._request_fingerprinter: RequestFingerprinterProtocol | None = Noneself._stats: StatsCollector | None = None+ def _set_force_stop_callback(+ self,+ callback: Callable[[], Awaitable[None] | Deferred[None] | None] | None,+ ) -> None:+ self._force_stop_callback = callback+def _update_root_log_handler(self) -> None:if get_scrapy_root_handler() is not None:# scrapy root handler already installed: update it with new settings@@ -340,7 +350,7 @@ class Crawler:def _create_engine(self) -> ExecutionEngine:return ExecutionEngine(self, lambda _: self.stop_async())- def stop(self) -> Deferred[None]:+ def stop(self, *, mode: _StopMode = "graceful") -> Deferred[None]:"""Start a graceful stop of the crawler and return a deferred that isfired when the crawler is stopped."""warnings.warn(@@ -348,17 +358,42 @@ class Crawler:ScrapyDeprecationWarning,stacklevel=2,)- return deferred_from_coro(self.stop_async())+ return deferred_from_coro(self.stop_async(mode=mode))- async def stop_async(self) -> None:+ async def stop_async(self, *, mode: _StopMode = "graceful") -> None:"""Start a graceful stop of the crawler and complete when the crawler is stopped... versionadded:: 2.14"""- if self.crawling:- self.crawling = False- if self.engine.running:- await self.engine.stop_async()+ mode = _normalize_stop_mode(mode)+ was_crawling = self.crawling+ self.crawling = False++ if not was_crawling and mode == "graceful":+ return++ if mode == "force":+ if self._force_stop_callback is not None:+ await ensure_awaitable(self._force_stop_callback())+ return+ logger.warning(+ "Force stop requested, but no process-level force stop callback is available. Falling back to fast stop."+ )+ mode = "fast"++ if self._engine is None:+ return++ # During shutdown callbacks, graceful stop may be re-entered after+ # the engine has already switched to non-running state.+ if mode == "graceful" and not self._engine.running:+ return++ try:+ await self._engine.stop_async(mode=mode)+ except RuntimeError as exc:+ if str(exc) != "Engine not running":+ raise@staticmethoddef _get_component(@@ -581,13 +616,16 @@ class CrawlerRunner(CrawlerRunnerBase):self._active.discard(d)self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed- def stop(self) -> Deferred[Any]:+ def stop(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:"""Stops simultaneously all the crawling jobs taking place.Returns a deferred that is fired when they all have ended."""- return DeferredList(deferred_from_coro(c.stop_async()) for c in self.crawlers)+ mode = _normalize_stop_mode(mode)+ return DeferredList(+ deferred_from_coro(c.stop_async(mode=mode)) for c in self.crawlers+ )@inlineCallbacksdef join(self) -> Generator[Deferred[Any], Any, None]:@@ -705,15 +743,16 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):return task- async def stop(self) -> None:+ async def stop(self, *, mode: _StopMode = "graceful") -> None:"""Stops simultaneously all the crawling jobs taking place.Completes when they all have ended."""+ mode = _normalize_stop_mode(mode)if self.crawlers:await asyncio.wait(- [asyncio.create_task(c.stop_async()) for c in self.crawlers]+ [asyncio.create_task(c.stop_async(mode=mode)) for c in self.crawlers])async def join(self) -> None:@@ -736,6 +775,14 @@ class CrawlerProcessBase(CrawlerRunnerBase):configure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)+ def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler:+ crawler = super()._create_crawler(spidercls)+ crawler._set_force_stop_callback(self._force_stop)+ return crawler++ def _force_stop(self) -> None:+ self._stop_reactor()+@abstractmethoddef start(self, stop_after_crawl: bool = True, install_signal_handlers: bool = True@@ -769,16 +816,25 @@ class CrawlerProcessBase(CrawlerRunnerBase):def _signal_shutdown(self, signum: int, _: Any) -> None:from twisted.internet import reactor- install_shutdown_handlers(self._signal_kill)+ install_shutdown_handlers(self._signal_fast_shutdown)reactor.callFromThread(self._log_shutdown, signum)reactor.callFromThread(self._graceful_stop_reactor)+ def _signal_fast_shutdown(self, signum: int, _: Any) -> None:+ from twisted.internet import reactor++ install_shutdown_handlers(self._signal_kill)+ reactor.callFromThread(self._log_fast_shutdown, signum)+ reactor.callFromThread(self._fast_stop_reactor)+def _signal_kill(self, signum: int, _: Any) -> None:from twisted.internet import reactorinstall_shutdown_handlers(signal.SIG_IGN)reactor.callFromThread(self._log_kill, signum)- reactor.callFromThread(self._stop_reactor)+ # Give the log line a moment to actually reach its output before the+ # process exits, since nothing else delays that exit past this point.+ reactor.callLater(_DEFER_DELAY, self._stop_reactor)# Logging cannot happen in a signal handler: the interrupted code may be# in the middle of writing to the same stream, and writing to it again@@ -789,7 +845,15 @@ class CrawlerProcessBase(CrawlerRunnerBase):def _log_shutdown(signum: int) -> None:signame = signal_names[signum]logger.info(- "Received %(signame)s, shutting down gracefully. Send again to force ",+ "Received %(signame)s, shutting down gracefully. Send again to stop faster.",+ {"signame": signame},+ )++ @staticmethod+ def _log_fast_shutdown(signum: int) -> None:+ signame = signal_names[signum]+ logger.info(+ "Received %(signame)s twice, dropping downloader requests. Send again to force unclean shutdown",{"signame": signame},)@@ -797,7 +861,8 @@ class CrawlerProcessBase(CrawlerRunnerBase):def _log_kill(signum: int) -> None:signame = signal_names[signum]logger.info(- "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame}+ "Received %(signame)s three times, forcing unclean shutdown",+ {"signame": signame},)def _setup_reactor(self, install_signal_handlers: bool) -> None:@@ -841,13 +906,20 @@ class CrawlerProcessBase(CrawlerRunnerBase):)@abstractmethod- def _stop_dfd(self) -> Deferred[Any]:+ def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:raise NotImplementedError@inlineCallbacksdef _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]:try:- yield self._stop_dfd()+ yield self._stop_dfd(mode="graceful")+ finally:+ self._stop_reactor()++ @inlineCallbacks+ def _fast_stop_reactor(self) -> Generator[Deferred[Any], Any, None]:+ try:+ yield self._stop_dfd(mode="fast")finally:self._stop_reactor()@@ -900,10 +972,12 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):spidercls = self.spider_loader.load(spidercls)init_reactor = not self._initialized_reactorself._initialized_reactor = True- return Crawler(spidercls, self.settings, init_reactor=init_reactor)+ crawler = Crawler(spidercls, self.settings, init_reactor=init_reactor)+ crawler._set_force_stop_callback(self._force_stop)+ return crawler- def _stop_dfd(self) -> Deferred[Any]:- return self.stop()+ def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:+ return self.stop(mode=mode)def start(self, stop_after_crawl: bool = True, install_signal_handlers: bool = True@@ -1001,8 +1075,18 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):self._initialized_reactor = Trueself._reactorless_main_task: asyncio.Future[None] | None = None- def _stop_dfd(self) -> Deferred[Any]:- return deferred_from_coro(self.stop())+ def _force_stop(self) -> None:+ if self.settings.getbool("TWISTED_REACTOR_ENABLED"):+ self._stop_reactor()+ return++ if (loop := self._reactorless_loop) is None:+ return+ if (task := self._reactorless_main_task) is not None:+ loop.call_soon_threadsafe(task.cancel)++ def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:+ return deferred_from_coro(self.stop(mode=mode))def start(self, stop_after_crawl: bool = True, install_signal_handlers: bool = True@@ -1151,24 +1235,32 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):})- def _signal_shutdown_reactorless(self, signum: int, _: Any) -> None:- install_shutdown_handlers(self._signal_kill_reactorless)+ def _schedule_reactorless_shutdown(self, *, mode: _StopMode, signum: int) -> None:if (loop := self._reactorless_loop) is None:return- loop.call_soon_threadsafe(self._log_shutdown, signum)- loop.call_soon_threadsafe(self._create_shutdown_task)+ log_func = self._log_fast_shutdown if mode == "fast" else self._log_shutdown+ loop.call_soon_threadsafe(log_func, signum)- def _create_shutdown_task(self) -> None:- assert self._reactorless_loop- coro = self._shutdown_graceful_reactorless()- try:- self._reactorless_loop.create_task(coro)- except RuntimeError:- coro.close()+ def _create_shutdown_task() -> None:+ coro = self._shutdown_reactorless(mode=mode)+ try:+ loop.create_task(coro)+ except RuntimeError:+ coro.close()++ loop.call_soon_threadsafe(_create_shutdown_task)++ def _signal_shutdown_reactorless(self, signum: int, _: Any) -> None:+ install_shutdown_handlers(self._signal_fast_shutdown_reactorless)+ self._schedule_reactorless_shutdown(mode="graceful", signum=signum)- async def _shutdown_graceful_reactorless(self) -> None:- await self.stop()+ def _signal_fast_shutdown_reactorless(self, signum: int, _: Any) -> None:+ install_shutdown_handlers(self._signal_kill_reactorless)+ self._schedule_reactorless_shutdown(mode="fast", signum=signum)++ async def _shutdown_reactorless(self, *, mode: _StopMode) -> None:+ await self.stop(mode=mode)if not self._stop_after_crawl:# wait until crawl tasks finish and cancel the futureawait self.join()@@ -1181,7 +1273,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):returnloop.call_soon_threadsafe(self._log_kill, signum)if (task := self._reactorless_main_task) is not None:- loop.call_soon_threadsafe(task.cancel)+ # Give the log line a moment to actually reach its output before+ # the process exits, since nothing else delays that exit past+ # this point.+ loop.call_later(_DEFER_DELAY, task.cancel)def _start_twisted(self, stop_after_crawl: bool, install_signal_handlers: booldiff --git a/scrapy/utils/_stopmode.py b/scrapy/utils/_stopmode.pynew file mode 100644index 000000000..ec4d083a6--- /dev/null+++ b/scrapy/utils/_stopmode.py@@ -0,0 +1,27 @@+from __future__ import annotations++from typing import Literal++_StopMode = Literal["graceful", "fast", "force"]++_STOP_MODE_PRIORITY: dict[_StopMode, int] = {+ "graceful": 0,+ "fast": 1,+ "force": 2,+}+++def _normalize_stop_mode(mode: _StopMode, *, allow_force: bool = True) -> _StopMode:+ if mode not in _STOP_MODE_PRIORITY:+ raise ValueError(+ f"Unknown stop mode {mode!r}. Expected one of: graceful, fast, force"+ )+ if mode == "force" and not allow_force:+ raise ValueError("The force stop mode is not supported in this context")+ return mode+++def _max_stop_mode(mode1: _StopMode, mode2: _StopMode) -> _StopMode:+ if _STOP_MODE_PRIORITY[mode1] >= _STOP_MODE_PRIORITY[mode2]:+ return mode1+ return mode2diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 584961379..8b497dc2a 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -6,6 +6,7 @@ from __future__ import annotationsimport asyncioimport inspect+import sysimport warningsfrom asyncio import Futurefrom collections import deque@@ -97,6 +98,30 @@ async def _process_pending_io() -> None:await sleep(0)+async def _process_pending_io_before_callback() -> None:+ """Same as :func:`_process_pending_io`, for use right before a spider+ callback, where a real delay is needed on Windows.++ Console control events, which is what Ctrl-C and Ctrl-Break send, are+ delivered on a separate thread there, and their Python-level handler only+ runs once the main thread reaches a bytecode boundary, which it cannot do+ while parked in the blocking wait of the reactor or the event loop, as+ nothing interrupts that wait. So a signal sent during a slow callback is+ only handled at the next wake-up, e.g. the engine heartbeat 5 seconds+ later, which is late enough to miss a shutdown entirely. Yielding for a+ moment keeps a timed call pending, which bounds that wait, and by the time+ it elapses a shutdown is under way and keeps the loop busy on its own.++ A signal wake-up socket on Windows would remove the need for this:+ https://github.com/python/cpython/issues/67246+ https://github.com/python/cpython/issues/86849+ """+ if sys.platform == "win32":+ await sleep(_DEFER_DELAY)+ return+ await _process_pending_io()++def defer_result(result: Any) -> Deferred[Any]: # pragma: no coverwarnings.warn("scrapy.utils.defer.defer_result() is deprecated, use"diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.py b/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.pynew file mode 100644index 000000000..d3eb2bec1--- /dev/null+++ b/tests/AsyncCrawlerProcess/reactorless_sleeping_no_stop_after_crawl.py@@ -0,0 +1,20 @@+import asyncio+import sys++import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class SleepingSpider(scrapy.Spider):+ name = "sleeping"++ start_urls = ["data:,;"]++ async def parse(self, response):+ await asyncio.sleep(int(sys.argv[1]))+++process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})++process.crawl(SleepingSpider)+process.start(stop_after_crawl=False)diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 034a4b034..dccb63bfd 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -2,10 +2,12 @@ from __future__ import annotationsimport warningsfrom typing import TYPE_CHECKING, Any, cast+from unittest.mock import patchimport OpenSSL.SSLimport pytestfrom pytest_twisted import async_yield_fixture+from twisted.internet.defer import CancelledError, Deferredfrom twisted.internet.endpoints import HostnameEndpointfrom twisted.internet.protocol import Factoryfrom twisted.internet.protocol import Protocol as TxProtocol@@ -33,7 +35,7 @@ from scrapy.core.downloader.contextfactory import (_ScrapyClientContextFactory,)from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import DownloadCancelledError, ScrapyDeprecationWarningfrom scrapy.resolver import dnscachefrom scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONNfrom scrapy.utils.defer import maybe_deferred_to_future@@ -50,8 +52,8 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from pathlib import Path- from twisted.internet.defer import Deferredfrom twisted.internet.interfaces import IConsumer, IListeningPort+ from twisted.python.failure import Failurefrom twisted.web.iweb import IBodyProducerfrom scrapy.http import Response@@ -157,6 +159,29 @@ class TestJitterSetting:assert jitter == pytest.approx(0.2)+class TestDownloaderStop:+ @coroutine_test+ async def test_waits_for_cancelled_downloads_to_settle(self) -> None:+ """stop() cancels in-flight downloads before returning.++ The engine relies on their cleanup (removal from ``_download_tasks``)+ having already happened by the time it returns, or it may consider the+ crawl finished while a download is still being torn down.+ """+ crawler = get_crawler(DefaultSpider)+ downloader = Downloader(crawler)++ requests = [Request(f"data:,{i}") for i in range(5)]+ for request in requests:+ download_dfd: Deferred[None] = Deferred()+ download_dfd.addBoth(downloader._download_task_done, request)+ downloader._download_tasks[request] = download_dfd++ await downloader.stop()++ assert not downloader._download_tasks++@pytest.mark.requires_reactor # this test is related to the Twisted HTTP codeclass TestContextFactoryBase:@async_yield_fixture@@ -578,6 +603,119 @@ async def test_fetch_deprecated_spider_arg():await crawler.crawl_async()+@coroutine_test+async def test_stop_drops_queued_requests() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ downloader = Downloader(crawler)+ slot = Slot(concurrency=1, delay=0, jitter=0)+ downloader.slots["example.com"] = slot++ request = Request("https://example.com")+ queue_dfd: Deferred[Any] = Deferred()+ failures: list[Failure] = []+ queue_dfd.addErrback(failures.append)+ slot.queue.append((request, queue_dfd))++ dropped = await downloader.stop()+ assert dropped == 1+ assert len(failures) == 1+ assert failures[0].check(DownloadCancelledError)+++@coroutine_test+async def test_stop_rejects_new_requests() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ downloader = Downloader(crawler)++ await downloader.stop()++ with pytest.raises(+ DownloadCancelledError,+ match="not accepting new requests",+ ):+ await downloader._enqueue_request(Request("https://example.com"))+++@coroutine_test+async def test_wait_for_download_errbacks_queue_deferred_on_error() -> None:+ crawler = get_crawler(DefaultSpider)+ downloader = Downloader(crawler)+ slot = Slot(concurrency=1, delay=0, jitter=0)++ queue_dfd: Deferred[Any] = Deferred()+ failures: list[Failure] = []+ queue_dfd.addErrback(failures.append)++ with patch.object(downloader, "_download", side_effect=RuntimeError("boom")):+ await downloader._wait_for_download(+ slot,+ Request("https://example.com"),+ queue_dfd,+ )++ assert len(failures) == 1+ assert failures[0].check(RuntimeError)+++@coroutine_test+async def test_wait_for_download_keeps_called_queue_deferred_on_error() -> None:+ crawler = get_crawler(DefaultSpider)+ downloader = Downloader(crawler)+ slot = Slot(concurrency=1, delay=0, jitter=0)++ queue_dfd: Deferred[Any] = Deferred()+ queue_dfd.callback(None)++ with patch.object(downloader, "_download", side_effect=RuntimeError("boom")):+ await downloader._wait_for_download(+ slot,+ Request("https://example.com"),+ queue_dfd,+ )++ assert queue_dfd.called+++@coroutine_test+async def test_stop_skips_called_queued_deferred() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ downloader = Downloader(crawler)+ slot = Slot(concurrency=1, delay=0, jitter=0)+ downloader.slots["example.com"] = slot++ queue_dfd: Deferred[Any] = Deferred()+ queue_dfd.callback(None)+ slot.queue.append((Request("https://example.com"), queue_dfd))++ dropped = await downloader.stop()+ assert dropped == 1+++@coroutine_test+async def test_stop_cancels_pending_download_tasks() -> None:+ crawler = get_crawler(DefaultSpider)+ downloader = Downloader(crawler)++ done_dfd: Deferred[None] = Deferred()+ done_dfd.callback(None)++ pending_dfd: Deferred[None] = Deferred()+ failures: list[Failure] = []+ pending_dfd.addErrback(failures.append)++ downloader._download_tasks[Request("https://done.example")] = done_dfd+ downloader._download_tasks[Request("https://pending.example")] = pending_dfd++ dropped = await downloader.stop()++ assert dropped == 1+ assert len(failures) == 1+ assert failures[0].check(CancelledError)++def test_deprecated_tls_module_names() -> None:with pytest.warns(ScrapyDeprecationWarning,diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 9bdaa6f38..a37aee4e7 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -6,8 +6,8 @@ import reimport signalimport threadingfrom pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar-from unittest.mock import MagicMock+from typing import TYPE_CHECKING, Any, ClassVar, cast+from unittest.mock import MagicMock, patchimport pytest@@ -17,7 +17,7 @@ from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcessfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settings-from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.defer import _DEFER_DELAY, maybe_deferred_to_futurefrom scrapy.utils.log import (_uninstall_scrapy_root_handler,configure_logging,@@ -665,9 +665,10 @@ class TestAsyncCrawlerProcessReactorlessHelpers:process, installed_handlers = self._bare_process(monkeypatch)process._reactorless_loop = None# No loop to schedule the shutdown task on, so it returns early, but it- # must still escalate the handler so a second signal forces a kill.+ # must still escalate the handler so a second signal forces a fast+ # shutdown.process._signal_shutdown_reactorless(signal.SIGINT, None)- assert installed_handlers == [process._signal_kill_reactorless]+ assert installed_handlers == [process._signal_fast_shutdown_reactorless]def test_signal_kill_reactorless_without_loop(self, monkeypatch: pytest.MonkeyPatch@@ -721,14 +722,19 @@ class TestAsyncCrawlerProcessReactorlessHelpers:process._signal_kill_reactorless(signal.SIGINT, None)assert log_calls == []loop.call_soon_threadsafe.assert_any_call(process._log_kill, signal.SIGINT)+ # The cancellation itself is delayed, giving the log line above a+ # moment to actually reach its output before the process exits.+ loop.call_later.assert_any_call(+ _DEFER_DELAY, process._reactorless_main_task.cancel+ )- def test_shutdown_graceful_reactorless_main_task_already_done(+ def test_shutdown_reactorless_main_task_already_done(self, monkeypatch: pytest.MonkeyPatch) -> None:process, _ = self._bare_process(monkeypatch)process._stop_after_crawl = False- async def noop() -> None:+ async def noop(*args: Any, **kwargs: Any) -> None:return Nonemonkeypatch.setattr(process, "stop", noop)@@ -741,25 +747,13 @@ class TestAsyncCrawlerProcessReactorlessHelpers:main_task.set_result(None)process._reactorless_main_task = main_task# The main task is already done, so it is not cancelled.- loop.run_until_complete(process._shutdown_graceful_reactorless())+ loop.run_until_complete(process._shutdown_reactorless(mode="graceful"))assert not main_task.cancelled()finally:loop.close()self._run_in_thread(run)- def test_create_shutdown_task_closed_loop(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- process, _ = self._bare_process(monkeypatch)- loop = asyncio.new_event_loop()- loop.close()- process._reactorless_loop = loop- process._stop_after_crawl = True- # create_task() raises RuntimeError on a closed loop; the coroutine- # must be closed instead of leaking.- process._create_shutdown_task()-def test_cancel_all_tasks_logs_task_exception(self) -> None:contexts: list[dict[str, Any]] = []task_was_cancelled: list[bool] = []@@ -843,7 +837,7 @@ class TestCrawlerProcessBaseSignalHandlers:process._signal_kill(signal.SIGINT, None)assert log_calls == []reactor.callFromThread.assert_any_call(process._log_kill, signal.SIGINT)- reactor.callFromThread.assert_any_call(process._stop_reactor)+ reactor.callLater.assert_any_call(_DEFER_DELAY, process._stop_reactor)@pytest.mark.parametrize(@@ -882,3 +876,282 @@ async def test_deprecated_crawler_stop() -> None:ScrapyDeprecationWarning, match=r"Crawler.stop\(\) is deprecated"):await maybe_deferred_to_future(crawler.stop())+++@coroutine_test+async def test_crawler_stop_async_invalid_mode() -> None:+ crawler = get_crawler(DefaultSpider)+ with pytest.raises(ValueError, match=r"Unknown stop mode"):+ await crawler.stop_async(mode="invalid") # type: ignore[arg-type]+++@coroutine_test+async def test_crawler_graceful_stop_non_running_engine_is_noop() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.crawling = True++ class DummyEngine:+ running = False+ called = False++ async def stop_async(self, *, mode: str = "graceful") -> None:+ self.called = True++ dummy_engine = DummyEngine()+ crawler.engine = dummy_engine # type: ignore[assignment]++ await crawler.stop_async(mode="graceful")++ assert dummy_engine.called is False+++@coroutine_test+async def test_crawler_force_stop_falls_back_to_fast(+ caplog: pytest.LogCaptureFixture,+) -> None:+ crawler = get_crawler(DefaultSpider)++ class DummyEngine:+ called_mode: str | None = None++ async def stop_async(self, *, mode: str = "graceful") -> None:+ self.called_mode = mode++ dummy_engine = DummyEngine()+ crawler.engine = dummy_engine # type: ignore[assignment]++ with caplog.at_level(logging.WARNING):+ await crawler.stop_async(mode="force")++ assert dummy_engine.called_mode == "fast"+ assert "Falling back to fast stop" in caplog.text+++@coroutine_test+async def test_crawler_force_stop_uses_force_callback() -> None:+ crawler = get_crawler(DefaultSpider)+ called = False++ def force_stop_callback() -> None:+ nonlocal called+ called = True++ crawler._set_force_stop_callback(force_stop_callback)+ await crawler.stop_async(mode="force")+ assert called+++@coroutine_test+async def test_crawler_stop_async_without_engine_is_noop() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.crawling = True++ await crawler.stop_async(mode="graceful")++ assert crawler.crawling is False+++@coroutine_test+async def test_crawler_stop_async_ignores_engine_not_running_runtime_error() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.crawling = True++ class DummyEngine:+ running = True+ called = False++ async def stop_async(self, *, mode: str = "graceful") -> None:+ self.called = True+ raise RuntimeError("Engine not running")++ dummy_engine = DummyEngine()+ crawler.engine = dummy_engine # type: ignore[assignment]++ await crawler.stop_async(mode="graceful")++ assert dummy_engine.called is True+++@coroutine_test+async def test_crawler_stop_async_reraises_other_runtime_errors() -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.crawling = True++ class DummyEngine:+ running = True++ async def stop_async(self, *, mode: str = "graceful") -> None:+ raise RuntimeError("different runtime error")++ crawler.engine = DummyEngine() # type: ignore[assignment]++ with pytest.raises(RuntimeError, match="different runtime error"):+ await crawler.stop_async(mode="graceful")++[email protected]_reactor+@coroutine_test+async def test_crawler_process_force_stop_via_public_crawler_api() -> None:+ crawler_process = CrawlerProcess({"LOG_INSTALL_ROOT_HANDLER": False})+ called = False++ def stop_reactor(_: Any = None) -> None:+ nonlocal called+ called = True++ crawler_process._stop_reactor = stop_reactor # type: ignore[method-assign]+ crawler = crawler_process.create_crawler(DefaultSpider)++ await crawler.stop_async(mode="force")++ assert called++[email protected]_asyncio[email protected]_reactor+@coroutine_test+async def test_async_crawler_process_force_stop_reactor_enabled_via_public_crawler_api() -> (+ None+):+ crawler_process = AsyncCrawlerProcess(+ {"TWISTED_REACTOR_ENABLED": True, "LOG_INSTALL_ROOT_HANDLER": False},+ )+ called = False++ def stop_reactor(_: Any = None) -> None:+ nonlocal called+ called = True++ crawler_process._stop_reactor = stop_reactor # type: ignore[method-assign]+ crawler = crawler_process.create_crawler(DefaultSpider)++ await crawler.stop_async(mode="force")++ assert called++[email protected]_asyncio+@coroutine_test+async def test_async_crawler_process_force_stop_reactorless_without_main_task(+ reactor_pytest: str,+) -> None:+ if reactor_pytest != "none":+ pytest.skip("This test is only for --reactor=none")++ crawler_process = AsyncCrawlerProcess(+ {"TWISTED_REACTOR_ENABLED": False, "LOG_INSTALL_ROOT_HANDLER": False},+ )+ assert crawler_process._reactorless_loop is not None+ assert crawler_process._reactorless_main_task is None+ crawler = crawler_process.create_crawler(DefaultSpider)++ await crawler.stop_async(mode="force")++[email protected]_asyncio+@coroutine_test+async def test_async_crawler_process_force_stop_reactorless_without_loop(+ reactor_pytest: str,+) -> None:+ if reactor_pytest != "none":+ pytest.skip("This test is only for --reactor=none")++ crawler_process = AsyncCrawlerProcess(+ {"TWISTED_REACTOR_ENABLED": False, "LOG_INSTALL_ROOT_HANDLER": False},+ )+ crawler_process._reactorless_loop = None+ crawler_process._reactorless_main_task = None+ crawler = crawler_process.create_crawler(DefaultSpider)++ await crawler.stop_async(mode="force")++[email protected]_asyncio+@coroutine_test+async def test_async_crawler_process_force_stop_reactorless_with_task(+ reactor_pytest: str,+) -> None:+ if reactor_pytest != "none":+ pytest.skip("This test is only for --reactor=none")++ crawler_process = AsyncCrawlerProcess(+ {"TWISTED_REACTOR_ENABLED": False, "LOG_INSTALL_ROOT_HANDLER": False},+ )++ class DummyLoop:+ callback = None++ def call_soon_threadsafe(self, callback) -> None:+ self.callback = callback++ class DummyTask:+ called = False++ def cancel(self) -> None:+ self.called = True++ loop = DummyLoop()+ task = DummyTask()+ crawler_process._reactorless_loop = cast("asyncio.AbstractEventLoop", loop)+ crawler_process._reactorless_main_task = cast("asyncio.Future[None]", task)+ crawler = crawler_process.create_crawler(DefaultSpider)++ await crawler.stop_async(mode="force")++ assert loop.callback is not None+ loop.callback()+ assert task.called is True+++def test_async_crawler_process_schedule_reactorless_shutdown_without_loop() -> None:+ crawler_process = object.__new__(AsyncCrawlerProcess)+ crawler_process._reactorless_loop = None++ crawler_process._schedule_reactorless_shutdown(+ mode="graceful", signum=signal.SIGINT+ )+++def test_async_crawler_process_schedule_reactorless_shutdown_runtime_error() -> None:+ crawler_process = object.__new__(AsyncCrawlerProcess)++ class DummyLoop:+ scheduled = False+ create_task_called = False++ def call_soon_threadsafe(self, callback, *args) -> None:+ self.scheduled = True+ callback(*args)++ def create_task(self, coro) -> None:+ self.create_task_called = True+ raise RuntimeError("event loop is closing")++ class DummyCoro:+ closed = False++ def close(self) -> None:+ self.closed = True++ loop = DummyLoop()+ coro = DummyCoro()+ called_mode: str | None = None++ def shutdown_reactorless(*, mode: str) -> DummyCoro:+ nonlocal called_mode+ called_mode = mode+ return coro++ crawler_process._reactorless_loop = cast("asyncio.AbstractEventLoop", loop)+ with patch.object(+ crawler_process, "_shutdown_reactorless", new=shutdown_reactorless+ ):+ crawler_process._schedule_reactorless_shutdown(+ mode="graceful", signum=signal.SIGINT+ )++ assert called_mode == "graceful"++ assert loop.scheduled+ assert loop.create_task_called+ assert coro.closeddiff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 7f9831958..fbcfc69e5 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -257,10 +257,15 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):p.expect_exact("Crawled (200)")p.kill(sig)p.expect_exact("shutting down gracefully")- # sending the second signal too fast often causes problems- await sleep(0.01)+ # Sending a new signal too fast often causes problems, e.g. on+ # Windows, where signal delivery is slower and more variable than on+ # POSIX.+ await sleep(0.1)p.kill(sig)- p.expect_exact("forcing unclean shutdown")+ p.expect_exact("dropping downloader requests")+ await sleep(0.1)+ p.kill(sig)+ p.expect_exact("forcing unclean shutdown", timeout=20)stop_spawn(p)@coroutine_test@@ -270,6 +275,26 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):def test_shutdown_graceful_no_stop(self) -> None:self._test_shutdown_graceful("sleeping.py", "--no-stop")+ async def _test_shutdown_fast(+ self, script: str = "sleeping.py", *extra_args: str+ ) -> None:+ sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]+ args = self.get_script_args(script, "3", *extra_args)+ p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())+ p.expect_exact("Spider opened")+ p.expect_exact("Crawled (200)")+ p.kill(sig)+ p.expect_exact("shutting down gracefully")+ await sleep(0.1)+ p.kill(sig)+ p.expect_exact("dropping downloader requests")+ p.expect_exact("Spider closed (shutdown)")+ stop_spawn(p)++ @coroutine_test+ async def test_shutdown_fast_no_stop(self) -> None:+ await self._test_shutdown_fast("sleeping.py", "--no-stop")+class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):@property@@ -482,6 +507,9 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):def test_reactorless_shutdown_graceful(self) -> None:self._test_shutdown_graceful("reactorless_sleeping.py")+ def test_shutdown_graceful_stop_after_crawl_false(self) -> None:+ self._test_shutdown_graceful("reactorless_sleeping_no_stop_after_crawl.py")+@coroutine_testasync def test_reactorless_shutdown_forced(self) -> None:await self._test_shutdown_forced("reactorless_sleeping.py")diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 86831f8da..f80eb9f5c 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -5,18 +5,30 @@ import loggingimport subprocessimport sysfrom typing import TYPE_CHECKING, Any-from unittest.mock import Mock+from unittest.mock import AsyncMock, Mock, patchimport pytest+from twisted.internet import defer+from twisted.python.failure import Failurefrom scrapy import signalsfrom scrapy.core.downloader import Downloaderfrom scrapy.core.engine import ExecutionEngine, _Slotfrom scrapy.core.scheduler import BaseScheduler-from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest+from scrapy.exceptions import (+ CloseSpider,+ DontCloseSpider,+ DownloadCancelledError,+ IgnoreRequest,+)from scrapy.http import Requestfrom scrapy.spiders import Spider-from scrapy.utils.defer import _schedule_coro, deferred_from_coro+from scrapy.utils.asyncio import sleep+from scrapy.utils.defer import (+ _schedule_coro,+ deferred_from_coro,+ maybe_deferred_to_future,+)from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -137,6 +149,71 @@ class TestEngine(TestEngineBase):yield deferred_from_coro(e.start_async())yield deferred_from_coro(e.stop_async())+ @coroutine_test+ async def test_stop_async_force_mode_not_supported(self) -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)++ with pytest.raises(ValueError, match="force stop mode is not supported"):+ await engine.stop_async(mode="force")++ @coroutine_test+ async def test_stop_async_not_running_raises(self) -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)++ with pytest.raises(RuntimeError, match="Engine not running"):+ await engine.stop_async()++ @coroutine_test+ async def test_stop_async_reentrant_fast_waits_for_closewait(self) -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)+ engine.spider = Mock()+ engine._stopping = True+ engine._closewait = defer.Deferred()++ with patch.object(+ engine, "close_spider_async", new_callable=AsyncMock+ ) as close:+ stop_dfd = deferred_from_coro(engine.stop_async(mode="fast"))+ await sleep(0)+ close.assert_called_once_with(reason="shutdown", mode="fast")+ assert not stop_dfd.called++ assert engine._closewait+ engine._closewait.callback(None)+ await maybe_deferred_to_future(stop_dfd)++ @coroutine_test+ async def test_stop_async_reentrant_graceful_without_spider_or_closewait(+ self,+ ) -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)+ engine._stopping = True++ with patch.object(+ engine, "close_spider_async", new_callable=AsyncMock+ ) as close:+ await engine.stop_async(mode="graceful")++ close.assert_not_called()++ @coroutine_test+ async def test_handle_downloader_output_ignores_fast_cancelled_failures(+ self,+ ) -> None:+ engine = ExecutionEngine(get_crawler(DefaultSpider), lambda _: None)+ engine.spider = Mock()+ engine._stop_mode = "fast"++ enqueue_scrape = Mock()+ engine.scraper.enqueue_scrape = enqueue_scrape # type: ignore[method-assign]++ result = Failure(DownloadCancelledError("dropped during fast stop"))+ await maybe_deferred_to_future(+ engine._handle_downloader_output(result, Request("https://example.com"))+ )++ enqueue_scrape.assert_not_called()+@pytest.mark.only_asyncio@coroutine_testasync def test_start_already_running_exception_asyncio(self):diff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.pyindex 8e9c9bd43..df572e4e5 100644--- a/tests/test_engine_close_spider.py+++ b/tests/test_engine_close_spider.py@@ -1,6 +1,7 @@from __future__ import annotations-from typing import TYPE_CHECKING, cast+from typing import TYPE_CHECKING, Any, cast+from unittest.mock import patchimport pytestfrom twisted.internet import defer@@ -151,3 +152,71 @@ async def test_exception_async_callback(await engine.open_spider_async()await engine.close_spider_async()assert "Error running spider_closed_callback" in caplog.text+++@coroutine_test+async def test_fast_close_stops_downloader_and_records_dropped_requests(+ crawler: Crawler,+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()++ calls = 0++ async def fast_stop_downloader() -> int:+ nonlocal calls+ calls += 1+ return 3++ with patch.object(engine.downloader, "stop", fast_stop_downloader):+ await engine.close_spider_async(mode="fast")++ assert calls == 1+ assert crawler.stats+ assert crawler.stats.get_value("downloader/request_dropped_count") == 3+++@coroutine_test+async def test_fast_close_tolerates_downloader_without_stop(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()++ downloader = engine.downloader++ class NoStopDownloader:+ def __getattr__(self, name: str) -> Any:+ if name == "stop":+ raise AttributeError(name)+ return getattr(downloader, name)++ engine.downloader = NoStopDownloader() # type: ignore[assignment]+ await engine.close_spider_async(mode="fast")++ assert "does not implement stop()" in caplog.text+ assert crawler.stats+ assert crawler.stats.get_value("downloader/request_dropped_count") is None+++@coroutine_test+async def test_fast_stop_downloader_is_idempotent(crawler: Crawler) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()++ calls = 0++ async def fast_stop_downloader() -> int:+ nonlocal calls+ calls += 1+ return 1++ with patch.object(engine.downloader, "stop", fast_stop_downloader):+ await engine._fast_stop_downloader()+ await engine._fast_stop_downloader()++ assert calls == 1+ await engine.close_spider_async()