Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

set asyncio event loop at start of new thread

Alex8b09b0e0d7eb
diff --git a/scrapy/shell.py b/scrapy/shell.py
index f2dff2ae3..515b71bb6 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
class Shell:
@@ -76,6 +77,10 @@ class Shell:
banner=self.vars.pop('banner', ''))
def _schedule(self, request, spider):
+ if is_asyncio_reactor_installed():
+ # set the asyncio event loop for the current thread
+ event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+ set_asyncio_event_loop(event_loop_path)
spider = self._open_spider(request, spider)
d = _request_deferred(request)
d.addCallback(lambda x: (x, spider))
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ddf354d88..532d7b0d0 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None):
reactor_class = load_object(reactor_path)
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
with suppress(error.ReactorAlreadyInstalledError):
- policy = get_asyncio_event_loop_policy()
- if event_loop_path is not None:
- event_loop_class = load_object(event_loop_path)
- event_loop = event_loop_class()
- asyncio.set_event_loop(event_loop)
- else:
- event_loop = policy.get_event_loop()
-
+ event_loop = set_asyncio_event_loop(event_loop_path)
asyncioreactor.install(eventloop=event_loop)
else:
*module, _ = reactor_path.split(".")
@@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None):
with suppress(error.ReactorAlreadyInstalledError):
installer()
+def set_asyncio_event_loop(event_loop_path):
+ """Sets and returns the event loop with specified import path."""
+ policy = get_asyncio_event_loop_policy()
+ if event_loop_path is not None:
+ event_loop_class = load_object(event_loop_path)
+ event_loop = event_loop_class()
+ asyncio.set_event_loop(event_loop)
+ else:
+ try:
+ event_loop = policy.get_event_loop()
+ except RuntimeError:
+ # `get_event_loop` is expected to fail when called from a new thread
+ # with no asyncio event loop yet installed. Such is the case when
+ # called from `scrapy shell`
+ event_loop = policy.new_event_loop()
+ asyncio.set_event_loop(event_loop)
+ return event_loop
+
def verify_installed_reactor(reactor_path):
"""Raises :exc:`Exception` if the installed