Scrapy, a fast high-level web crawling & scraping framework for Python.
Don't start the engine in the shell. (#8179)
Andrey Rakhmatullinfe30c1882a12
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 3574d8e62..532edd4f8 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -10,12 +10,10 @@ import asynciofrom threading import Threadfrom typing import TYPE_CHECKING, Any, ClassVar-from scrapy import signalsfrom scrapy.commands import ScrapyCommandfrom scrapy.crawler import AsyncCrawlerProcess, Crawlerfrom scrapy.http import Requestfrom scrapy.shell import Shell-from scrapy.utils.defer import _schedule_corofrom scrapy.utils.spider import DefaultSpider, spidercls_for_requestfrom scrapy.utils.url import guess_scheme@@ -97,31 +95,17 @@ class Command(ScrapyCommand):shell.start(url=url, redirect=not opts.no_redirect)def _init_with_reactor(self, crawler: Crawler) -> None:- # Create the engine and run start_async() in the main thread+ # Create the engine in the main threadcrawler.engine = crawler._create_engine()- _schedule_coro(crawler.engine.start_async(_start_request_processing=False))self._start_crawler_thread()def _init_without_reactor(self, crawler: Crawler) -> None:- # Create the engine and run start_async() in the event loop thread+ # Create the engine in the event loop threadloop = self._get_reactorless_loop()self._start_crawler_thread()async def _init_engine() -> None:- started: asyncio.Future[None] = loop.create_future()-- def on_engine_started(**kwargs: Any) -> None:- started.set_result(None)-- crawler.signals.connect(on_engine_started, signals.engine_started)crawler.engine = crawler._create_engine()- loop.create_task(- crawler.engine.start_async(_start_request_processing=False)- )- # Wait until the engine has been started, so that the engine is- # usable when this returns to the main thread.- await started- crawler.signals.disconnect(on_engine_started, signals.engine_started)future = asyncio.run_coroutine_threadsafe(_init_engine(), loop)future.result()diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 4b8575a28..1ac33bea9 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -182,11 +182,9 @@ class ExecutionEngine:ScrapyDeprecationWarning,stacklevel=2,)- return deferred_from_coro(- self.start_async(_start_request_processing=_start_request_processing)- )+ return deferred_from_coro(self.start_async())- async def start_async(self, *, _start_request_processing: bool = True) -> None:+ async def start_async(self) -> None:"""Start the execution engine... versionadded:: 2.14@@ -199,19 +197,18 @@ class ExecutionEngine:if self._stopping:# band-aid until https://github.com/scrapy/scrapy/issues/6916return- if _start_request_processing and self.spider is None:- # require an opened spider when not run in scrapy shell+ if self.spider is None:+ # require an opened spiderreturnself.running = Trueself._closewait = Deferred()- if _start_request_processing:- coro = self._start_request_processing()- if is_asyncio_available():- # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470- # (can happen when this is cancelled, e.g. in test_close_during_start_iteration())- self._start_request_processing_awaitable = asyncio.ensure_future(coro)- else:- self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)+ coro = self._start_request_processing()+ if is_asyncio_available():+ # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470+ # (can happen when this is cancelled, e.g. in test_close_during_start_iteration())+ self._start_request_processing_awaitable = asyncio.ensure_future(coro)+ else:+ self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)with contextlib.suppress(asyncio.exceptions.CancelledError):await maybe_deferred_to_future(self._closewait)diff --git a/scrapy/shell.py b/scrapy/shell.pyindex f22c80167..324a9bc76 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -22,6 +22,7 @@ import scrapyfrom scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningfrom scrapy.http import Request, Response+from scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils._shell import DEFAULT_PYTHON_SHELLS, start_python_console@@ -51,12 +52,11 @@ if TYPE_CHECKING:# reactor to be in a separate thread: the shell sends the request to# the reactor and waits for the result synchronously).#-# Thus the only thing Shell needs an event loop for is fetch(). More machinery-# is used for it to work. In chronological order:-# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine, then-# calls-# _schedule_coro(crawler.engine.start_async(_start_request_processing=False)),-# which initializes the engine but doesn't start processing of requests.+# Thus the only thing Shell needs an event loop for is fetch(). The engine is+# never started as the shell doesn't need request processing and+# download_async() still works.+# In chronological order:+# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine.# 2. scrapy.commands.shell.Command.run() calls crawler_process.start() in a# thread which starts a reactor in that thread.# 3. When fetch() is called, it prepares a request and calls Shell._schedule()@@ -67,7 +67,7 @@ if TYPE_CHECKING:# 6. Shell._schedule() calls engine.download_async(request), which completes# with the response.#-# In the reactorless mode this is slightly different, the engine initialization+# In the reactorless mode this is slightly different, the engine creation# happens in the event loop thread as many things need either a reactor or a# running event loop.#@@ -76,12 +76,8 @@ if TYPE_CHECKING:# immediately installs a reactor (which is maybe not thread-specific?) or an# event loop (which *is* thread-specific, so the main thread will always have# a (not running) loop installed.-# * scrapy.commands.shell.Command.run() calls _schedule_coro() in the main-# thread, and various engine init code also calls similar things,-# conceptually this shouldn't work (and doesn't in the reactorless mode, so-# there the initialization is moved to the event loop thread).-# * The engine has several code paths specifically for the shell, and the shell-# uses several private members of the engine and of AsyncCrawlerProcess.+# * The shell uses several private members of Crawler (_create_engine(),+# _create_spider()) and of AsyncCrawlerProcess (_reactorless_loop).class Shell:@@ -179,7 +175,7 @@ class Shell:)async def _schedule(self, request: Request, spider: Spider | None) -> Response:- """Send the request to the engine, wait for the result.+ """Download the request through the engine and return the response.Runs in the reactor thread when using the reactor, or in the asyncioevent loop thread otherwise.@@ -205,14 +201,17 @@ class Shell:) -> None:if isinstance(request_or_url, Request):request = request_or_url- if request.callback or request.errback:+ if (+ not (request.callback is None or request.callback is NO_CALLBACK)+ or request.errback+ ):warnings.warn("Callbacks and errbacks of Request objects passed to fetch() are ignored.",stacklevel=2,)else:url = any_to_uri(request_or_url)- request = Request(url, dont_filter=True, **kwargs)+ request = Request(url, callback=NO_CALLBACK, dont_filter=True, **kwargs)if not redirect:request.meta["dont_redirect"] = Trueresponse: Response | None = Nonediff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex b101e732b..3550265e8 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -176,6 +176,10 @@ class TestShellCommand:_, out, _ = proc("shell", "-c", code)assert url in out+ def test_engine_not_started(self) -> None:+ _, out, _ = proc("shell", "-c", "crawler.engine.running")+ assert out.strip() == "False"+class TestShellCommandWithSpider(TestProjectBase):@pytest.fixture(autouse=True)diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 5c73ad35d..14e58a975 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -449,16 +449,8 @@ async def test_stop_without_spider_closes_downloader() -> None:crawler = get_crawler(DefaultSpider)engine = crawler.engine = ExecutionEngine(crawler, lambda _: None)engine.downloader.close = Mock(wraps=engine.downloader.close) # type: ignore[method-assign]- started: defer.Deferred[None] = defer.Deferred()-- def on_engine_started(**kwargs: Any) -> None:- started.callback(None)-- crawler.signals.connect(on_engine_started, signals.engine_started)- start_dfd = deferred_from_coro(engine.start_async(_start_request_processing=False))- await maybe_deferred_to_future(started)+ await engine.start_async()await engine.stop_async()- await maybe_deferred_to_future(start_dfd)engine.downloader.close.assert_called_once()diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.pyindex aa0149ff2..e8065f35b 100644--- a/tests/test_extension_memusage.py+++ b/tests/test_extension_memusage.py@@ -2,7 +2,8 @@ from __future__ import annotationsimport loggingimport sys-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Any+from unittest.mock import AsyncMock, MagicMockimport pytest@@ -15,11 +16,10 @@ from scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.utils import OneShotLoop-from tests.utils.cmdline import procfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from tests.mockserver.http import MockServer+ from collections.abc import Coroutine# MemoryUsage relies on the stdlib 'resource' module (not available on Windows)pytestmark = pytest.mark.skipif(@@ -64,19 +64,25 @@ def test_memusage_disabled() -> None:build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))-def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:- # The Scrapy shell starts the engine without opening a spider, so the- # whole crawler is stopped instead of a spider being closed.- _, out, err = proc(- "shell",- mockserver.url("/text"),- "-c",- "response.status",- "--set",- "MEMUSAGE_LIMIT_MB=1",- )- assert "Memory usage exceeded 1MiB" in err- assert "200" in out+def test_memusage_limit_stops_crawler_without_spider(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """When the limit is exceeded while no spider is open (e.g. it was+ already closed), the whole crawler is stopped instead."""+ crawler = get_crawler(settings_dict={"MEMUSAGE_LIMIT_MB": 1})+ crawler.engine = MagicMock(spider=None)+ crawler.stop_async = AsyncMock() # type: ignore[method-assign]+ scheduled: list[Coroutine[Any, Any, None]] = []+ monkeypatch.setattr(memusage_mod, "_schedule_coro", scheduled.append)+ monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 2 * MB)+ ext = build_from_crawler(MemoryUsage, crawler)++ ext._check_limit()++ assert crawler.stats.get_value("memusage/limit_reached") == 1+ crawler.stop_async.assert_called_once_with()+ for coro in scheduled:+ coro.close()@coroutine_test