Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into change-init-order

Andrey Rakhmatullinb4380995da15
diff --cc scrapy/crawler.py
index 44ffc44ce,3c9f28a00..974754964
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@@ -104,9 -105,26 +104,10 @@@ class Crawler
crawler=self,
)
- reactor_class: str = self.settings["TWISTED_REACTOR"]
- event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
- if init_reactor:
- # this needs to be done after the spider settings are merged,
- # but before something imports twisted.internet.reactor
- if reactor_class:
- install_reactor(reactor_class, event_loop)
- else:
- from twisted.internet import reactor # noqa: F401
- log_reactor_info()
- if reactor_class:
- verify_installed_reactor(reactor_class)
- if is_asyncio_reactor_installed() and event_loop:
- verify_installed_asyncio_event_loop(event_loop)
-
- self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
-
- self.settings.freeze()
+ self._init_reactor = init_reactor
+
self.crawling: bool = False
+ self._started: bool = False
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
diff --cc tests/test_crawler.py
index 1fe865ee4,067934961..120991ae7
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@@ -46,7 -68,28 +47,17 @@@ class CrawlerTestCase(BaseCrawlerTest)
with raises(ValueError):
Crawler(DefaultSpider())
+ @defer.inlineCallbacks
+ def test_crawler_crawl_twice_deprecated(self):
+ crawler = Crawler(NoRequestsSpider)
+ yield crawler.crawl()
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Running Crawler.crawl\(\) more than once is deprecated",
+ ):
+ yield crawler.crawl()
+
-class SpiderSettingsTestCase(unittest.TestCase):
- def test_spider_custom_settings(self):
- class MySpider(scrapy.Spider):
- name = "spider"
- custom_settings = {"AUTOTHROTTLE_ENABLED": True}
-
- crawler = get_crawler(MySpider)
- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
- self.assertIn(AutoThrottle, enabled_exts)
-
-
class CrawlerLoggingTestCase(unittest.TestCase):
def test_no_root_handler_installed(self):
handler = get_scrapy_root_handler()