Scrapy, a fast high-level web crawling & scraping framework for Python.
Wait for engine_started in the reactorless shell init (#8168)
Andrey Rakhmatullin9008b6dbfc8b
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 9c6bfb2f9..35a3ed423 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -10,6 +10,7 @@ import asynciofrom threading import Threadfrom typing import TYPE_CHECKING, Any, ClassVar+from scrapy import signalsfrom scrapy.commands import ScrapyCommandfrom scrapy.crawler import AsyncCrawlerProcess, Crawlerfrom scrapy.http import Request@@ -106,13 +107,20 @@ class Command(ScrapyCommand):self._start_crawler_thread()async def _init_engine() -> None:- # We may need to wait until some parts of start_async() have- # finished, which may need a special event in the engine and may- # wait until https://github.com/scrapy/scrapy/issues/6916+ started: asyncio.Future[None] = loop.create_future()++ def on_engine_started(**kwargs: Any) -> None:+ started.set_result(None)++ crawler.signals.connect(on_engine_started, signals.engine_started)crawler.engine = crawler._create_engine()loop.create_task(crawler.engine.start_async(_start_request_processing=False))+ # Wait until the engine has been started, so that the engine is+ # usable when this returns to the main thread.+ await started+ crawler.signals.disconnect(on_engine_started, signals.engine_started)future = asyncio.run_coroutine_threadsafe(_init_engine(), loop)future.result()