Scrapy, a fast high-level web crawling & scraping framework for Python.
Log Scrapy info after add-ons, document extending LOG_VERSIONS (#8246)
Adrian84a82af4c803
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 2a0d8522e..788de9d1f 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -141,6 +141,19 @@ Set some basic configuration::meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict`to avoid mistakes.+Log the versions of the packages the add-on uses, unless the user has set+:setting:`LOG_VERSIONS`:++.. code-block:: python++ from scrapy.settings import SETTINGS_PRIORITIES+++ class MyAddon:+ def update_settings(self, settings):+ if settings.getpriority("LOG_VERSIONS") == SETTINGS_PRIORITIES["default"]:+ settings.add_to_list("LOG_VERSIONS", "boto")+Check dependencies:.. code-block:: pythondiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 723f70fc1..4b83e47c1 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -178,6 +178,7 @@ class Crawler:returnself.addons.load_settings(self.settings)+ log_scrapy_info(self.settings)self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")self._apply_deprecated_spider_attr("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"@@ -776,7 +777,6 @@ class CrawlerProcessBase(CrawlerRunnerBase):super().__init__(settings)self._reactor_crawler: Crawler | None = Noneconfigure_logging(self.settings, install_root_handler)- log_scrapy_info(self.settings)def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler:crawler = super()._create_crawler(spidercls)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex a37aee4e7..8f56f8921 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -16,7 +16,7 @@ from scrapy import Spiderfrom scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcessfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottle-from scrapy.settings import Settings, default_settings+from scrapy.settings import SETTINGS_PRIORITIES, Settings, default_settingsfrom scrapy.utils.defer import _DEFER_DELAY, maybe_deferred_to_futurefrom scrapy.utils.log import (_uninstall_scrapy_root_handler,@@ -840,27 +840,47 @@ class TestCrawlerProcessBaseSignalHandlers:reactor.callLater.assert_any_call(_DEFER_DELAY, process._stop_reactor)+class _AddVersionAddon:+ def update_settings(self, settings: Settings) -> None:+ settings.set("BOT_NAME", "addonbot", priority="addon")+ if settings.getpriority("LOG_VERSIONS") == SETTINGS_PRIORITIES["default"]:+ settings.add_to_list("LOG_VERSIONS", "itemadapter")++@pytest.mark.parametrize(- ("settings", "items"),+ ("settings", "bot", "items"),[- ({}, default_settings.LOG_VERSIONS),- ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),- ({"LOG_VERSIONS": []}, None),+ ({}, "scrapybot", default_settings.LOG_VERSIONS),+ ({"LOG_VERSIONS": ["itemadapter"]}, "scrapybot", ["itemadapter"]),+ ({"LOG_VERSIONS": []}, "scrapybot", None),+ (+ {"ADDONS": {_AddVersionAddon: 0}},+ "addonbot",+ [*default_settings.LOG_VERSIONS, "itemadapter"],+ ),+ (+ {"ADDONS": {_AddVersionAddon: 0}, "LOG_VERSIONS": ["w3lib"]},+ "addonbot",+ ["w3lib"],+ ),],)def test_log_scrapy_info(- settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture+ settings: dict[str, Any],+ bot: str,+ items: list[str] | None,+ caplog: pytest.LogCaptureFixture,) -> None:- with caplog.at_level("INFO"):- CrawlerProcess({**settings, "LOG_INSTALL_ROOT_HANDLER": False})+ with caplog.at_level("INFO", logger="scrapy.utils.log"):+ get_crawler(settings_dict=settings)+ records = [r for r in caplog.records if r.name == "scrapy.utils.log"]assert (- caplog.records[0].getMessage()- == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"- ), repr(caplog.records[0].msg)+ records[0].getMessage() == f"Scrapy {scrapy.__version__} started (bot: {bot})"+ )if not items:- assert len(caplog.records) == 1+ assert len(records) == 1return- version_string = caplog.records[1].getMessage()+ version_string = records[1].getMessage()expected_items_pattern = "',\n '".join(f"{item}': '[^']+('\n +'[^']+)*" for item in items)