Scrapy, a fast high-level web crawling & scraping framework for Python.
set asyncio event loop at start of new thread
Alex8b09b0e0d7eb
diff --git a/scrapy/shell.py b/scrapy/shell.pyindex f2dff2ae3..515b71bb6 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_consolefrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.misc import load_objectfrom scrapy.utils.response import open_in_browser+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loopclass Shell:@@ -76,6 +77,10 @@ class Shell:banner=self.vars.pop('banner', ''))def _schedule(self, request, spider):+ if is_asyncio_reactor_installed():+ # set the asyncio event loop for the current thread+ event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']+ set_asyncio_event_loop(event_loop_path)spider = self._open_spider(request, spider)d = _request_deferred(request)d.addCallback(lambda x: (x, spider))diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex ddf354d88..532d7b0d0 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None):reactor_class = load_object(reactor_path)if reactor_class is asyncioreactor.AsyncioSelectorReactor:with suppress(error.ReactorAlreadyInstalledError):- policy = get_asyncio_event_loop_policy()- if event_loop_path is not None:- event_loop_class = load_object(event_loop_path)- event_loop = event_loop_class()- asyncio.set_event_loop(event_loop)- else:- event_loop = policy.get_event_loop()-+ event_loop = set_asyncio_event_loop(event_loop_path)asyncioreactor.install(eventloop=event_loop)else:*module, _ = reactor_path.split(".")@@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None):with suppress(error.ReactorAlreadyInstalledError):installer()+def set_asyncio_event_loop(event_loop_path):+ """Sets and returns the event loop with specified import path."""+ policy = get_asyncio_event_loop_policy()+ if event_loop_path is not None:+ event_loop_class = load_object(event_loop_path)+ event_loop = event_loop_class()+ asyncio.set_event_loop(event_loop)+ else:+ try:+ event_loop = policy.get_event_loop()+ except RuntimeError:+ # `get_event_loop` is expected to fail when called from a new thread+ # with no asyncio event loop yet installed. Such is the case when+ # called from `scrapy shell`+ event_loop = policy.new_event_loop()+ asyncio.set_event_loop(event_loop)+ return event_loop+def verify_installed_reactor(reactor_path):"""Raises :exc:`Exception` if the installed