Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'origin/master' into change-init-order
Andrey Rakhmatullinb4380995da15
diff --cc scrapy/crawler.pyindex 44ffc44ce,3c9f28a00..974754964--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@@ -104,9 -105,26 +104,10 @@@ class Crawlercrawler=self,)- reactor_class: str = self.settings["TWISTED_REACTOR"]- event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]- if init_reactor:- # this needs to be done after the spider settings are merged,- # but before something imports twisted.internet.reactor- if reactor_class:- install_reactor(reactor_class, event_loop)- else:- from twisted.internet import reactor # noqa: F401- log_reactor_info()- if reactor_class:- verify_installed_reactor(reactor_class)- if is_asyncio_reactor_installed() and event_loop:- verify_installed_asyncio_event_loop(event_loop)-- self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)-- self.settings.freeze()+ self._init_reactor = init_reactor+self.crawling: bool = False+ self._started: bool = Falseself.spider: Optional[Spider] = Noneself.engine: Optional[ExecutionEngine] = Nonediff --cc tests/test_crawler.pyindex 1fe865ee4,067934961..120991ae7--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@@ -46,7 -68,28 +47,17 @@@ class CrawlerTestCase(BaseCrawlerTest)with raises(ValueError):Crawler(DefaultSpider())+ @defer.inlineCallbacks+ def test_crawler_crawl_twice_deprecated(self):+ crawler = Crawler(NoRequestsSpider)+ yield crawler.crawl()+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Running Crawler.crawl\(\) more than once is deprecated",+ ):+ yield crawler.crawl()+-class SpiderSettingsTestCase(unittest.TestCase):- def test_spider_custom_settings(self):- class MySpider(scrapy.Spider):- name = "spider"- custom_settings = {"AUTOTHROTTLE_ENABLED": True}-- crawler = get_crawler(MySpider)- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]- self.assertIn(AutoThrottle, enabled_exts)--class CrawlerLoggingTestCase(unittest.TestCase):def test_no_root_handler_installed(self):handler = get_scrapy_root_handler()