Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow customizing logged software versions (#6582)

Adrián Chaves21b9ba717c16
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cce4a7b3e..76904a26e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1228,6 +1228,25 @@ Default: ``False``
If ``True``, the logs will just contain the root path. If it is set to ``False``
then it displays the component responsible for the log output
+.. setting:: LOG_VERSIONS
+
+LOG_VERSIONS
+------------
+
+Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]``
+
+Logs the installed versions of the specified items.
+
+An item can be any installed Python package.
+
+The following special items are also supported:
+
+- ``libxml2``
+
+- ``Platform`` (:func:`platform.platform`)
+
+- ``Python``
+
.. setting:: LOGSTATS_INTERVAL
LOGSTATS_INTERVAL
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 571f4fda8..713a78ad9 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -2,7 +2,7 @@ import argparse
import scrapy
from scrapy.commands import ScrapyCommand
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
class Command(ScrapyCommand):
@@ -26,7 +26,7 @@ class Command(ScrapyCommand):
def run(self, args: list[str], opts: argparse.Namespace) -> None:
if opts.verbose:
- versions = scrapy_components_versions()
+ versions = get_versions()
width = max(len(n) for (n, _) in versions)
for name, version in versions:
print(f"{name:<{width}} : {version}")
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 89ab21fbe..0bbde118e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG"
LOG_FILE = None
LOG_FILE_APPEND = True
LOG_SHORT_NAMES = False
+LOG_VERSIONS = [
+ "lxml",
+ "libxml2",
+ "cssselect",
+ "parsel",
+ "w3lib",
+ "Twisted",
+ "Python",
+ "pyOpenSSL",
+ "cryptography",
+ "Platform",
+]
SCHEDULER_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6165d1f72..d51231b82 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
+import pprint
import sys
from collections.abc import MutableMapping
from logging.config import dictConfig
@@ -12,7 +13,7 @@ from twisted.python.failure import Failure
import scrapy
from scrapy.settings import Settings, _SettingsKeyT
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
@@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None:
"Scrapy %(version)s started (bot: %(bot)s)",
{"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
)
- versions = [
- f"{name} {version}"
- for name, version in scrapy_components_versions()
- if name != "Scrapy"
- ]
- logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
+ software = settings.getlist("LOG_VERSIONS")
+ if not software:
+ return
+ versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)
+ logger.info(f"Versions:\n{versions}")
def log_reactor_info() -> None:
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 996a5cdb3..ff1f9b346 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -1,31 +1,46 @@
+from __future__ import annotations
+
import platform
import sys
+from importlib.metadata import version
+from warnings import warn
-import cryptography
-import cssselect
import lxml.etree
-import parsel
-import twisted
-import w3lib
-import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings.default_settings import LOG_VERSIONS
from scrapy.utils.ssl import get_openssl_version
+_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS
+
+
+def _version(item):
+ lowercase_item = item.lower()
+ if lowercase_item == "libxml2":
+ return ".".join(map(str, lxml.etree.LIBXML_VERSION))
+ if lowercase_item == "platform":
+ return platform.platform()
+ if lowercase_item == "pyopenssl":
+ return get_openssl_version()
+ if lowercase_item == "python":
+ return sys.version.replace("\n", "- ")
+ return version(item)
+
+
+def get_versions(
+ software: list | None = None,
+) -> list[tuple[str, str]]:
+ software = software or _DEFAULT_SOFTWARE
+ return [(item, _version(item)) for item in software]
+
def scrapy_components_versions() -> list[tuple[str, str]]:
- lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
- libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
- return [
- ("Scrapy", scrapy.__version__),
- ("lxml", lxml_version),
- ("libxml2", libxml2_version),
- ("cssselect", cssselect.__version__),
- ("parsel", parsel.__version__),
- ("w3lib", w3lib.__version__),
- ("Twisted", twisted.version.short()),
- ("Python", sys.version.replace("\n", "- ")),
- ("pyOpenSSL", get_openssl_version()),
- ("cryptography", cryptography.__version__),
- ("Platform", platform.platform()),
- ]
+ warn(
+ (
+ "scrapy.utils.versions.scrapy_components_versions() is deprecated, "
+ "use scrapy.utils.versions.get_versions() instead."
+ ),
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return get_versions()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a77531f62..f3e5ebf5d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,6 +1,7 @@
import logging
import os
import platform
+import re
import signal
import subprocess
import sys
@@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
log,
)
self.assertIn("DEBUG: Using asyncio event loop", log)
+
+
+ ["settings", "items"],
+ (
+ ({}, default_settings.LOG_VERSIONS),
+ ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
+ ({"LOG_VERSIONS": []}, None),
+ ),
+)
+def test_log_scrapy_info(settings, items, caplog):
+ with caplog.at_level("INFO"):
+ CrawlerProcess(settings)
+ assert (
+ caplog.records[0].getMessage()
+ == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"
+ ), repr(caplog.records[0].msg)
+ if not items:
+ assert len(caplog.records) == 1
+ return
+ version_string = caplog.records[1].getMessage()
+ expected_items_pattern = "',\n '".join(
+ f"{item}': '[^']+('\n +'[^']+)*" for item in items
+ )
+ assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)