Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't start the engine in the shell. (#8179)

Andrey Rakhmatullinfe30c1882a12
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 3574d8e62..532edd4f8 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -10,12 +10,10 @@ import asyncio
from threading import Thread
from typing import TYPE_CHECKING, Any, ClassVar
-from scrapy import signals
from scrapy.commands import ScrapyCommand
from scrapy.crawler import AsyncCrawlerProcess, Crawler
from scrapy.http import Request
from scrapy.shell import Shell
-from scrapy.utils.defer import _schedule_coro
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
@@ -97,31 +95,17 @@ class Command(ScrapyCommand):
shell.start(url=url, redirect=not opts.no_redirect)
def _init_with_reactor(self, crawler: Crawler) -> None:
- # Create the engine and run start_async() in the main thread
+ # Create the engine in the main thread
crawler.engine = crawler._create_engine()
- _schedule_coro(crawler.engine.start_async(_start_request_processing=False))
self._start_crawler_thread()
def _init_without_reactor(self, crawler: Crawler) -> None:
- # Create the engine and run start_async() in the event loop thread
+ # Create the engine in the event loop thread
loop = self._get_reactorless_loop()
self._start_crawler_thread()
async def _init_engine() -> None:
- started: asyncio.Future[None] = loop.create_future()
-
- def on_engine_started(**kwargs: Any) -> None:
- started.set_result(None)
-
- crawler.signals.connect(on_engine_started, signals.engine_started)
crawler.engine = crawler._create_engine()
- loop.create_task(
- crawler.engine.start_async(_start_request_processing=False)
- )
- # Wait until the engine has been started, so that the engine is
- # usable when this returns to the main thread.
- await started
- crawler.signals.disconnect(on_engine_started, signals.engine_started)
future = asyncio.run_coroutine_threadsafe(_init_engine(), loop)
future.result()
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4b8575a28..1ac33bea9 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -182,11 +182,9 @@ class ExecutionEngine:
ScrapyDeprecationWarning,
stacklevel=2,
)
- return deferred_from_coro(
- self.start_async(_start_request_processing=_start_request_processing)
- )
+ return deferred_from_coro(self.start_async())
- async def start_async(self, *, _start_request_processing: bool = True) -> None:
+ async def start_async(self) -> None:
"""Start the execution engine.
.. versionadded:: 2.14
@@ -199,19 +197,18 @@ class ExecutionEngine:
if self._stopping:
# band-aid until https://github.com/scrapy/scrapy/issues/6916
return
- if _start_request_processing and self.spider is None:
- # require an opened spider when not run in scrapy shell
+ if self.spider is None:
+ # require an opened spider
return
self.running = True
self._closewait = Deferred()
- if _start_request_processing:
- coro = self._start_request_processing()
- if is_asyncio_available():
- # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470
- # (can happen when this is cancelled, e.g. in test_close_during_start_iteration())
- self._start_request_processing_awaitable = asyncio.ensure_future(coro)
- else:
- self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)
+ coro = self._start_request_processing()
+ if is_asyncio_available():
+ # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470
+ # (can happen when this is cancelled, e.g. in test_close_during_start_iteration())
+ self._start_request_processing_awaitable = asyncio.ensure_future(coro)
+ else:
+ self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)
with contextlib.suppress(asyncio.exceptions.CancelledError):
await maybe_deferred_to_future(self._closewait)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index f22c80167..324a9bc76 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -22,6 +22,7 @@ import scrapy
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.http import Request, Response
+from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils._shell import DEFAULT_PYTHON_SHELLS, start_python_console
@@ -51,12 +52,11 @@ if TYPE_CHECKING:
# reactor to be in a separate thread: the shell sends the request to
# the reactor and waits for the result synchronously).
#
-# Thus the only thing Shell needs an event loop for is fetch(). More machinery
-# is used for it to work. In chronological order:
-# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine, then
-# calls
-# _schedule_coro(crawler.engine.start_async(_start_request_processing=False)),
-# which initializes the engine but doesn't start processing of requests.
+# Thus the only thing Shell needs an event loop for is fetch(). The engine is
+# never started as the shell doesn't need request processing and
+# download_async() still works.
+# In chronological order:
+# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine.
# 2. scrapy.commands.shell.Command.run() calls crawler_process.start() in a
# thread which starts a reactor in that thread.
# 3. When fetch() is called, it prepares a request and calls Shell._schedule()
@@ -67,7 +67,7 @@ if TYPE_CHECKING:
# 6. Shell._schedule() calls engine.download_async(request), which completes
# with the response.
#
-# In the reactorless mode this is slightly different, the engine initialization
+# In the reactorless mode this is slightly different, the engine creation
# happens in the event loop thread as many things need either a reactor or a
# running event loop.
#
@@ -76,12 +76,8 @@ if TYPE_CHECKING:
# immediately installs a reactor (which is maybe not thread-specific?) or an
# event loop (which *is* thread-specific, so the main thread will always have
# a (not running) loop installed.
-# * scrapy.commands.shell.Command.run() calls _schedule_coro() in the main
-# thread, and various engine init code also calls similar things,
-# conceptually this shouldn't work (and doesn't in the reactorless mode, so
-# there the initialization is moved to the event loop thread).
-# * The engine has several code paths specifically for the shell, and the shell
-# uses several private members of the engine and of AsyncCrawlerProcess.
+# * The shell uses several private members of Crawler (_create_engine(),
+# _create_spider()) and of AsyncCrawlerProcess (_reactorless_loop).
class Shell:
@@ -179,7 +175,7 @@ class Shell:
)
async def _schedule(self, request: Request, spider: Spider | None) -> Response:
- """Send the request to the engine, wait for the result.
+ """Download the request through the engine and return the response.
Runs in the reactor thread when using the reactor, or in the asyncio
event loop thread otherwise.
@@ -205,14 +201,17 @@ class Shell:
) -> None:
if isinstance(request_or_url, Request):
request = request_or_url
- if request.callback or request.errback:
+ if (
+ not (request.callback is None or request.callback is NO_CALLBACK)
+ or request.errback
+ ):
warnings.warn(
"Callbacks and errbacks of Request objects passed to fetch() are ignored.",
stacklevel=2,
)
else:
url = any_to_uri(request_or_url)
- request = Request(url, dont_filter=True, **kwargs)
+ request = Request(url, callback=NO_CALLBACK, dont_filter=True, **kwargs)
if not redirect:
request.meta["dont_redirect"] = True
response: Response | None = None
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index b101e732b..3550265e8 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -176,6 +176,10 @@ class TestShellCommand:
_, out, _ = proc("shell", "-c", code)
assert url in out
+ def test_engine_not_started(self) -> None:
+ _, out, _ = proc("shell", "-c", "crawler.engine.running")
+ assert out.strip() == "False"
+
class TestShellCommandWithSpider(TestProjectBase):
@pytest.fixture(autouse=True)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 5c73ad35d..14e58a975 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -449,16 +449,8 @@ async def test_stop_without_spider_closes_downloader() -> None:
crawler = get_crawler(DefaultSpider)
engine = crawler.engine = ExecutionEngine(crawler, lambda _: None)
engine.downloader.close = Mock(wraps=engine.downloader.close) # type: ignore[method-assign]
- started: defer.Deferred[None] = defer.Deferred()
-
- def on_engine_started(**kwargs: Any) -> None:
- started.callback(None)
-
- crawler.signals.connect(on_engine_started, signals.engine_started)
- start_dfd = deferred_from_coro(engine.start_async(_start_request_processing=False))
- await maybe_deferred_to_future(started)
+ await engine.start_async()
await engine.stop_async()
- await maybe_deferred_to_future(start_dfd)
engine.downloader.close.assert_called_once()
diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py
index aa0149ff2..e8065f35b 100644
--- a/tests/test_extension_memusage.py
+++ b/tests/test_extension_memusage.py
@@ -2,7 +2,8 @@ from __future__ import annotations
import logging
import sys
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
+from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -15,11 +16,10 @@ from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils import OneShotLoop
-from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from tests.mockserver.http import MockServer
+ from collections.abc import Coroutine
# MemoryUsage relies on the stdlib 'resource' module (not available on Windows)
pytestmark = pytest.mark.skipif(
@@ -64,19 +64,25 @@ def test_memusage_disabled() -> None:
build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))
-def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:
- # The Scrapy shell starts the engine without opening a spider, so the
- # whole crawler is stopped instead of a spider being closed.
- _, out, err = proc(
- "shell",
- mockserver.url("/text"),
- "-c",
- "response.status",
- "--set",
- "MEMUSAGE_LIMIT_MB=1",
- )
- assert "Memory usage exceeded 1MiB" in err
- assert "200" in out
+def test_memusage_limit_stops_crawler_without_spider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """When the limit is exceeded while no spider is open (e.g. it was
+ already closed), the whole crawler is stopped instead."""
+ crawler = get_crawler(settings_dict={"MEMUSAGE_LIMIT_MB": 1})
+ crawler.engine = MagicMock(spider=None)
+ crawler.stop_async = AsyncMock() # type: ignore[method-assign]
+ scheduled: list[Coroutine[Any, Any, None]] = []
+ monkeypatch.setattr(memusage_mod, "_schedule_coro", scheduled.append)
+ monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 2 * MB)
+ ext = build_from_crawler(MemoryUsage, crawler)
+
+ ext._check_limit()
+
+ assert crawler.stats.get_value("memusage/limit_reached") == 1
+ crawler.stop_async.assert_called_once_with()
+ for coro in scheduled:
+ coro.close()
@coroutine_test