Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Log Scrapy info after add-ons, document extending LOG_VERSIONS (#8246)

Adrian84a82af4c803
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 2a0d8522e..788de9d1f 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -141,6 +141,19 @@ Set some basic configuration:
:meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict`
to avoid mistakes.
+Log the versions of the packages the add-on uses, unless the user has set
+:setting:`LOG_VERSIONS`:
+
+.. code-block:: python
+
+ from scrapy.settings import SETTINGS_PRIORITIES
+
+
+ class MyAddon:
+ def update_settings(self, settings):
+ if settings.getpriority("LOG_VERSIONS") == SETTINGS_PRIORITIES["default"]:
+ settings.add_to_list("LOG_VERSIONS", "boto")
+
Check dependencies:
.. code-block:: python
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 723f70fc1..4b83e47c1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -178,6 +178,7 @@ class Crawler:
return
self.addons.load_settings(self.settings)
+ log_scrapy_info(self.settings)
self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
self._apply_deprecated_spider_attr(
"max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"
@@ -776,7 +777,6 @@ class CrawlerProcessBase(CrawlerRunnerBase):
super().__init__(settings)
self._reactor_crawler: Crawler | None = None
configure_logging(self.settings, install_root_handler)
- log_scrapy_info(self.settings)
def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler:
crawler = super()._create_crawler(spidercls)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a37aee4e7..8f56f8921 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -16,7 +16,7 @@ from scrapy import Spider
from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
-from scrapy.settings import Settings, default_settings
+from scrapy.settings import SETTINGS_PRIORITIES, Settings, default_settings
from scrapy.utils.defer import _DEFER_DELAY, maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
@@ -840,27 +840,47 @@ class TestCrawlerProcessBaseSignalHandlers:
reactor.callLater.assert_any_call(_DEFER_DELAY, process._stop_reactor)
+class _AddVersionAddon:
+ def update_settings(self, settings: Settings) -> None:
+ settings.set("BOT_NAME", "addonbot", priority="addon")
+ if settings.getpriority("LOG_VERSIONS") == SETTINGS_PRIORITIES["default"]:
+ settings.add_to_list("LOG_VERSIONS", "itemadapter")
+
+
@pytest.mark.parametrize(
- ("settings", "items"),
+ ("settings", "bot", "items"),
[
- ({}, default_settings.LOG_VERSIONS),
- ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
- ({"LOG_VERSIONS": []}, None),
+ ({}, "scrapybot", default_settings.LOG_VERSIONS),
+ ({"LOG_VERSIONS": ["itemadapter"]}, "scrapybot", ["itemadapter"]),
+ ({"LOG_VERSIONS": []}, "scrapybot", None),
+ (
+ {"ADDONS": {_AddVersionAddon: 0}},
+ "addonbot",
+ [*default_settings.LOG_VERSIONS, "itemadapter"],
+ ),
+ (
+ {"ADDONS": {_AddVersionAddon: 0}, "LOG_VERSIONS": ["w3lib"]},
+ "addonbot",
+ ["w3lib"],
+ ),
],
)
def test_log_scrapy_info(
- settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture
+ settings: dict[str, Any],
+ bot: str,
+ items: list[str] | None,
+ caplog: pytest.LogCaptureFixture,
) -> None:
- with caplog.at_level("INFO"):
- CrawlerProcess({**settings, "LOG_INSTALL_ROOT_HANDLER": False})
+ with caplog.at_level("INFO", logger="scrapy.utils.log"):
+ get_crawler(settings_dict=settings)
+ records = [r for r in caplog.records if r.name == "scrapy.utils.log"]
assert (
- caplog.records[0].getMessage()
- == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"
- ), repr(caplog.records[0].msg)
+ records[0].getMessage() == f"Scrapy {scrapy.__version__} started (bot: {bot})"
+ )
if not items:
- assert len(caplog.records) == 1
+ assert len(records) == 1
return
- version_string = caplog.records[1].getMessage()
+ version_string = records[1].getMessage()
expected_items_pattern = "',\n '".join(
f"{item}': '[^']+('\n +'[^']+)*" for item in items
)