Scrapy, a fast high-level web crawling & scraping framework for Python.
Log when AsyncCrawlerProcess or CrawlerProcess is instantiated. (#7034)
Andrey Rakhmatullin80beec41b5cc
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex ef737b9c5..ef658e9b5 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -663,6 +663,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):):super().__init__(settings, install_root_handler)self._initialized_reactor: bool = False+ logger.debug("Using CrawlerProcess")def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:if isinstance(spidercls, str):@@ -737,6 +738,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):install_root_handler: bool = True,):super().__init__(settings, install_root_handler)+ logger.debug("Using AsyncCrawlerProcess")# We want the asyncio event loop to be installed early, so that it's# always the correct one. And as we do that, we can also install the# reactor here.diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 9ea893f05..9aaeb3e8d 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -131,30 +131,12 @@ class TestCommandCrawlerProcess(TestCommandBase):"""Test that the command uses the expected kind of *CrawlerProcessand produces expected errors when needed."""- name = "crawltest"-- NORMAL_MSG = "Type of self.crawler_process: <class 'scrapy.crawler.CrawlerProcess'>"- ASYNC_MSG = (- "Type of self.crawler_process: <class 'scrapy.crawler.AsyncCrawlerProcess'>"- )+ name = "crawl"+ NORMAL_MSG = "Using CrawlerProcess"+ ASYNC_MSG = "Using AsyncCrawlerProcess"def setup_method(self):super().setup_method()- (self.cwd / self.project_name / "commands").mkdir(exist_ok=True)- (self.cwd / self.project_name / "commands" / "__init__.py").touch()- (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text("""-from scrapy.commands.crawl import Command--class CrawlerProcessCrawlCommand(Command):- requires_project = True-- def run(self, args, opts):- print(f"Type of self.crawler_process: {type(self.crawler_process)}")- super().run(args, opts)-""")-- self._append_settings(f"COMMANDS_MODULE = '{self.project_name}.commands'\n")-(self.cwd / self.project_name / "spiders" / "sp.py").write_text("""import scrapy@@ -186,6 +168,8 @@ class MySpider(scrapy.Spider):yield""")+ self._append_settings("LOG_LEVEL = 'DEBUG'\n")+def _append_settings(self, text: str) -> None:"""Add text to the end of the project settings.py."""with (self.cwd / self.project_name / "settings.py").open(@@ -207,16 +191,16 @@ class MySpider(scrapy.Spider):def _assert_spider_works(self, msg: str, *args: str) -> None:"""The command uses the expected *CrawlerProcess, the spider works."""- _, out, err = self.proc(self.name, *args)- assert msg in out, out- assert "It works!" in err, err- assert "Spider closed (finished)" in err, err+ _, _, err = self.proc(self.name, *args)+ assert msg in err+ assert "It works!" in err+ assert "Spider closed (finished)" in errdef _assert_spider_asyncio_fail(self, msg: str, *args: str) -> None:"""The command uses the expected *CrawlerProcess, the spider fails to use asyncio."""- _, out, err = self.proc(self.name, *args)- assert msg in out, out- assert "no running event loop" in err, err+ _, _, err = self.proc(self.name, *args)+ assert msg in err+ assert "no running event loop" in errdef test_project_settings(self):"""The reactor is set via the project default settings (to the asyncio value).@@ -316,13 +300,13 @@ class MySpider(scrapy.Spider):spider,"{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",)- _, out, err = self.proc(self.name, spider)- assert self.ASYNC_MSG in out, out+ _, _, err = self.proc(self.name, spider)+ assert self.ASYNC_MSG in errassert ("The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"" does not match the requested one"" (twisted.internet.selectreactor.SelectReactor)"- ) in err, err+ ) in errdef test_project_asyncio_spider_settings_select_forced(self):"""The reactor is set via the project settings to the asyncio value