Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate Crawler.stop(). (#6999)
Andrey Rakhmatullin57f539d7a5b7
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 7fb906eab..c5f3b6f0c 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport asyncioimport loggingimport warnings+from collections.abc import AsyncIterator, Callable, Coroutine, Generatorfrom time import timefrom traceback import format_excfrom typing import TYPE_CHECKING, Any@@ -44,8 +45,6 @@ from scrapy.utils.python import global_object_namefrom scrapy.utils.reactor import CallLaterOnceif TYPE_CHECKING:- from collections.abc import AsyncIterator, Callable, Generator-from twisted.internet.task import LoopingCallfrom scrapy.core.downloader import Downloader@@ -102,7 +101,9 @@ class ExecutionEngine:def __init__(self,crawler: Crawler,- spider_closed_callback: Callable[[Spider], Deferred[None] | None],+ spider_closed_callback: Callable[+ [Spider], Coroutine[Any, Any, None] | Deferred[None] | None+ ],) -> None:self.crawler: Crawler = crawlerself.settings: Settings = crawler.settings@@ -113,9 +114,9 @@ class ExecutionEngine:self.spider: Spider | None = Noneself.running: bool = Falseself.paused: bool = False- self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = (- spider_closed_callback- )+ self._spider_closed_callback: Callable[+ [Spider], Coroutine[Any, Any, None] | Deferred[None] | None+ ] = spider_closed_callbackself.start_time: float | None = Noneself._start: AsyncIterator[Any] | None = Noneself._closewait: Deferred[None] | None = None@@ -625,7 +626,11 @@ class ExecutionEngine:self.spider = Nonetry:- if (d := self._spider_closed_callback(spider)) is not None:- await maybe_deferred_to_future(d)+ aw = self._spider_closed_callback(spider)+ # TODO: replace with ensure_awaitable() when we add it+ if isinstance(aw, Coroutine):+ await aw+ elif isinstance(aw, Deferred):+ await maybe_deferred_to_future(aw)except Exception:log_failure("Error running spider_closed_callback")diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 369dc2255..ef737b9c5 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -5,6 +5,7 @@ import contextlibimport loggingimport pprintimport signal+import warningsfrom abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, Any, TypeVar@@ -13,12 +14,13 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom scrapy import Spider, signalsfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEngine+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManagerfrom scrapy.settings import Settings, overridden_settingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loaderfrom scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.defer import deferred_from_coro, deferred_to_future+from scrapy.utils.defer import deferred_from_corofrom scrapy.utils.log import (LogCounterHandler,configure_logging,@@ -207,30 +209,28 @@ class Crawler:return self.spidercls.from_crawler(self, *args, **kwargs)def _create_engine(self) -> ExecutionEngine:- return ExecutionEngine(self, lambda _: self.stop())+ return ExecutionEngine(self, lambda _: self.stop_async())- @inlineCallbacks- def stop(self) -> Generator[Deferred[Any], Any, None]:+ def stop(self) -> Deferred[None]:"""Start a graceful stop of the crawler and return a deferred that isfired when the crawler is stopped."""- if self.crawling:- self.crawling = False- assert self.engine- if self.engine.running:- yield deferred_from_coro(self.engine.stop_async())+ warnings.warn(+ "Crawler.stop() is deprecated, use stop_async() instead",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return deferred_from_coro(self.stop_async())async def stop_async(self) -> None:"""Start a graceful stop of the crawler and complete when the crawler is stopped... versionadded:: VERSION-- This function requires- :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be- installed."""- if not is_asyncio_available():- raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.")- await deferred_to_future(self.stop())+ if self.crawling:+ self.crawling = False+ assert self.engine+ if self.engine.running:+ await self.engine.stop_async()@staticmethoddef _get_component(@@ -447,7 +447,7 @@ class CrawlerRunner(CrawlerRunnerBase):Returns a deferred that is fired when they all have ended."""- return DeferredList(c.stop() for c in self.crawlers)+ return DeferredList(deferred_from_coro(c.stop_async()) for c in self.crawlers)@inlineCallbacksdef join(self) -> Generator[Deferred[Any], Any, None]:diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex a345bbcc9..990c144b5 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -115,7 +115,7 @@ class MemoryUsage:self.crawler.engine.close_spider_async(reason="memusage_exceeded"))else:- self.crawler.stop()+ _schedule_coro(self.crawler.stop_async())else:logger.info("Peak memory usage is %(virtualsize)dMiB",diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 70674c021..de20d6b90 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -31,7 +31,11 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settingsfrom scrapy.utils.asyncio import call_later-from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro+from scrapy.utils.defer import (+ deferred_f_from_coro_f,+ deferred_from_coro,+ maybe_deferred_to_future,+)from scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_reactor_settings@@ -1174,3 +1178,14 @@ def test_log_scrapy_info(settings, items, caplog):f"{item}': '[^']+('\n +'[^']+)*" for item in items)assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)+++@deferred_f_from_coro_f+async def test_deprecated_crawler_stop() -> None:+ crawler = get_crawler(DefaultSpider)+ d = crawler.crawl()+ await maybe_deferred_to_future(d)+ with pytest.warns(+ ScrapyDeprecationWarning, match=r"Crawler.stop\(\) is deprecated"+ ):+ await maybe_deferred_to_future(crawler.stop())diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 94afcc2ad..2558b8dc6 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -180,12 +180,12 @@ class CrawlerRun:dispatcher.connect(self.stop, signals.engine_stopped)await maybe_deferred_to_future(self.deferred)- def stop(self):+ async def stop(self):for name, signal in vars(signals).items():if not name.startswith("_"):disconnect_all(signal)self.deferred.callback(None)- return self.crawler.stop()+ await self.crawler.stop_async()def geturl(self, path: str) -> str:return self.mockserver.url(path)@@ -726,3 +726,15 @@ class TestEngineCloseSpider:await engine.open_spider_async()await engine.close_spider_async()assert "Error running spider_closed_callback" in caplog.text++ @deferred_f_from_coro_f+ async def test_exception_async_callback(+ self, crawler: Crawler, caplog: pytest.LogCaptureFixture+ ) -> None:+ async def cb(_):+ raise ValueError++ engine = ExecutionEngine(crawler, cb)+ await engine.open_spider_async()+ await engine.close_spider_async()+ assert "Error running spider_closed_callback" in caplog.textdiff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex ef03771c3..ef1da9b3a 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -15,6 +15,7 @@ from scrapy.core.scheduler import BaseScheduler, Schedulerfrom scrapy.crawler import Crawlerfrom scrapy.http import Requestfrom scrapy.spiders import Spider+from scrapy.utils.defer import _schedule_corofrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawler@@ -114,7 +115,7 @@ class SchedulerHandler(ABC):def close_scheduler(self):self.scheduler.close("finished")- self.mock_crawler.stop()+ _schedule_coro(self.mock_crawler.stop_async())self.mock_crawler.engine.downloader.close()def setup_method(self):