Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move install_asyncio_reactor() from commands to CrawlerProcess.

Andrey Rakhmatullin855bbebc8bb8
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index b50761e4a..8093fd402 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,6 +1,5 @@
import os
from scrapy.commands import ScrapyCommand
-from scrapy.utils.asyncio import install_asyncio_reactor
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.python import without_none_values
from scrapy.exceptions import UsageError
@@ -27,8 +26,6 @@ class Command(ScrapyCommand):
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
- if self.settings.getbool('ASYNCIO_ENABLED'):
- install_asyncio_reactor()
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index bfe844eb5..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -2,7 +2,6 @@ import sys
import os
from importlib import import_module
-from scrapy.utils.asyncio import install_asyncio_reactor
from scrapy.utils.spider import iter_spider_classes
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
@@ -51,8 +50,6 @@ class Command(ScrapyCommand):
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
- if self.settings.getbool('ASYNCIO_ENABLED'):
- install_asyncio_reactor()
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 450260004..706c8a59d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -14,6 +14,7 @@ from scrapy.extension import ExtensionManager
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import install_asyncio_reactor
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
from scrapy.utils.log import (
@@ -256,6 +257,8 @@ class CrawlerProcess(CrawlerRunner):
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
+ if self.settings.getbool('ASYNCIO_ENABLED'):
+ install_asyncio_reactor()
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)