Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CrawlerProcess: initiate the reactor only once

Adrián Chaves3ecbea4b876e
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..9939a19eb 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -290,6 +290,7 @@ class CrawlerProcess(CrawlerRunner):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
+ self._initiated_reactor = False
def _signal_shutdown(self, signum, _):
from twisted.internet import reactor
@@ -310,7 +311,9 @@ class CrawlerProcess(CrawlerRunner):
def _create_crawler(self, spidercls):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
- return Crawler(spidercls, self.settings, init_reactor=True)
+ init_reactor = not self._initiated_reactor
+ self._initiated_reactor = True
+ return Crawler(spidercls, self.settings, init_reactor=init_reactor)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
"""