Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Log when AsyncCrawlerProcess or CrawlerProcess is instantiated. (#7034)

Andrey Rakhmatullin80beec41b5cc
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ef737b9c5..ef658e9b5 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -663,6 +663,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
):
super().__init__(settings, install_root_handler)
self._initialized_reactor: bool = False
+ logger.debug("Using CrawlerProcess")
def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:
if isinstance(spidercls, str):
@@ -737,6 +738,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
install_root_handler: bool = True,
):
super().__init__(settings, install_root_handler)
+ logger.debug("Using AsyncCrawlerProcess")
# We want the asyncio event loop to be installed early, so that it's
# always the correct one. And as we do that, we can also install the
# reactor here.
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 9ea893f05..9aaeb3e8d 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -131,30 +131,12 @@ class TestCommandCrawlerProcess(TestCommandBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
- name = "crawltest"
-
- NORMAL_MSG = "Type of self.crawler_process: <class 'scrapy.crawler.CrawlerProcess'>"
- ASYNC_MSG = (
- "Type of self.crawler_process: <class 'scrapy.crawler.AsyncCrawlerProcess'>"
- )
+ name = "crawl"
+ NORMAL_MSG = "Using CrawlerProcess"
+ ASYNC_MSG = "Using AsyncCrawlerProcess"
def setup_method(self):
super().setup_method()
- (self.cwd / self.project_name / "commands").mkdir(exist_ok=True)
- (self.cwd / self.project_name / "commands" / "__init__.py").touch()
- (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text("""
-from scrapy.commands.crawl import Command
-
-class CrawlerProcessCrawlCommand(Command):
- requires_project = True
-
- def run(self, args, opts):
- print(f"Type of self.crawler_process: {type(self.crawler_process)}")
- super().run(args, opts)
-""")
-
- self._append_settings(f"COMMANDS_MODULE = '{self.project_name}.commands'\n")
-
(self.cwd / self.project_name / "spiders" / "sp.py").write_text("""
import scrapy
@@ -186,6 +168,8 @@ class MySpider(scrapy.Spider):
yield
""")
+ self._append_settings("LOG_LEVEL = 'DEBUG'\n")
+
def _append_settings(self, text: str) -> None:
"""Add text to the end of the project settings.py."""
with (self.cwd / self.project_name / "settings.py").open(
@@ -207,16 +191,16 @@ class MySpider(scrapy.Spider):
def _assert_spider_works(self, msg: str, *args: str) -> None:
"""The command uses the expected *CrawlerProcess, the spider works."""
- _, out, err = self.proc(self.name, *args)
- assert msg in out, out
- assert "It works!" in err, err
- assert "Spider closed (finished)" in err, err
+ _, _, err = self.proc(self.name, *args)
+ assert msg in err
+ assert "It works!" in err
+ assert "Spider closed (finished)" in err
def _assert_spider_asyncio_fail(self, msg: str, *args: str) -> None:
"""The command uses the expected *CrawlerProcess, the spider fails to use asyncio."""
- _, out, err = self.proc(self.name, *args)
- assert msg in out, out
- assert "no running event loop" in err, err
+ _, _, err = self.proc(self.name, *args)
+ assert msg in err
+ assert "no running event loop" in err
def test_project_settings(self):
"""The reactor is set via the project default settings (to the asyncio value).
@@ -316,13 +300,13 @@ class MySpider(scrapy.Spider):
spider,
"{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",
)
- _, out, err = self.proc(self.name, spider)
- assert self.ASYNC_MSG in out, out
+ _, _, err = self.proc(self.name, spider)
+ assert self.ASYNC_MSG in err
assert (
"The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
" does not match the requested one"
" (twisted.internet.selectreactor.SelectReactor)"
- ) in err, err
+ ) in err
def test_project_asyncio_spider_settings_select_forced(self):
"""The reactor is set via the project settings to the asyncio value