Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow customizing logged software versions (#6582)
Adrián Chaves21b9ba717c16
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex cce4a7b3e..76904a26e 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1228,6 +1228,25 @@ Default: ``False``If ``True``, the logs will just contain the root path. If it is set to ``False``then it displays the component responsible for the log output+.. setting:: LOG_VERSIONS++LOG_VERSIONS+------------++Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]``++Logs the installed versions of the specified items.++An item can be any installed Python package.++The following special items are also supported:++- ``libxml2``++- ``Platform`` (:func:`platform.platform`)++- ``Python``+.. setting:: LOGSTATS_INTERVALLOGSTATS_INTERVALdiff --git a/scrapy/commands/version.py b/scrapy/commands/version.pyindex 571f4fda8..713a78ad9 100644--- a/scrapy/commands/version.py+++ b/scrapy/commands/version.py@@ -2,7 +2,7 @@ import argparseimport scrapyfrom scrapy.commands import ScrapyCommand-from scrapy.utils.versions import scrapy_components_versions+from scrapy.utils.versions import get_versionsclass Command(ScrapyCommand):@@ -26,7 +26,7 @@ class Command(ScrapyCommand):def run(self, args: list[str], opts: argparse.Namespace) -> None:if opts.verbose:- versions = scrapy_components_versions()+ versions = get_versions()width = max(len(n) for (n, _) in versions)for name, version in versions:print(f"{name:<{width}} : {version}")diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 89ab21fbe..0bbde118e 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG"LOG_FILE = NoneLOG_FILE_APPEND = TrueLOG_SHORT_NAMES = False+LOG_VERSIONS = [+ "lxml",+ "libxml2",+ "cssselect",+ "parsel",+ "w3lib",+ "Twisted",+ "Python",+ "pyOpenSSL",+ "cryptography",+ "Platform",+]SCHEDULER_DEBUG = Falsediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 6165d1f72..d51231b82 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -1,6 +1,7 @@from __future__ import annotationsimport logging+import pprintimport sysfrom collections.abc import MutableMappingfrom logging.config import dictConfig@@ -12,7 +13,7 @@ from twisted.python.failure import Failureimport scrapyfrom scrapy.settings import Settings, _SettingsKeyT-from scrapy.utils.versions import scrapy_components_versions+from scrapy.utils.versions import get_versionsif TYPE_CHECKING:@@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None:"Scrapy %(version)s started (bot: %(bot)s)",{"version": scrapy.__version__, "bot": settings["BOT_NAME"]},)- versions = [- f"{name} {version}"- for name, version in scrapy_components_versions()- if name != "Scrapy"- ]- logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})+ software = settings.getlist("LOG_VERSIONS")+ if not software:+ return+ versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)+ logger.info(f"Versions:\n{versions}")def log_reactor_info() -> None:diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.pyindex 996a5cdb3..ff1f9b346 100644--- a/scrapy/utils/versions.py+++ b/scrapy/utils/versions.py@@ -1,31 +1,46 @@+from __future__ import annotations+import platformimport sys+from importlib.metadata import version+from warnings import warn-import cryptography-import cssselectimport lxml.etree-import parsel-import twisted-import w3lib-import scrapy+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.settings.default_settings import LOG_VERSIONSfrom scrapy.utils.ssl import get_openssl_version+_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS+++def _version(item):+ lowercase_item = item.lower()+ if lowercase_item == "libxml2":+ return ".".join(map(str, lxml.etree.LIBXML_VERSION))+ if lowercase_item == "platform":+ return platform.platform()+ if lowercase_item == "pyopenssl":+ return get_openssl_version()+ if lowercase_item == "python":+ return sys.version.replace("\n", "- ")+ return version(item)+++def get_versions(+ software: list | None = None,+) -> list[tuple[str, str]]:+ software = software or _DEFAULT_SOFTWARE+ return [(item, _version(item)) for item in software]+def scrapy_components_versions() -> list[tuple[str, str]]:- lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))- libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))-- return [- ("Scrapy", scrapy.__version__),- ("lxml", lxml_version),- ("libxml2", libxml2_version),- ("cssselect", cssselect.__version__),- ("parsel", parsel.__version__),- ("w3lib", w3lib.__version__),- ("Twisted", twisted.version.short()),- ("Python", sys.version.replace("\n", "- ")),- ("pyOpenSSL", get_openssl_version()),- ("cryptography", cryptography.__version__),- ("Platform", platform.platform()),- ]+ warn(+ (+ "scrapy.utils.versions.scrapy_components_versions() is deprecated, "+ "use scrapy.utils.versions.get_versions() instead."+ ),+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return get_versions()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex a77531f62..f3e5ebf5d 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,6 +1,7 @@import loggingimport osimport platform+import reimport signalimport subprocessimport sys@@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):log,)self.assertIn("DEBUG: Using asyncio event loop", log)+++ ["settings", "items"],+ (+ ({}, default_settings.LOG_VERSIONS),+ ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),+ ({"LOG_VERSIONS": []}, None),+ ),+)+def test_log_scrapy_info(settings, items, caplog):+ with caplog.at_level("INFO"):+ CrawlerProcess(settings)+ assert (+ caplog.records[0].getMessage()+ == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"+ ), repr(caplog.records[0].msg)+ if not items:+ assert len(caplog.records) == 1+ return+ version_string = caplog.records[1].getMessage()+ expected_items_pattern = "',\n '".join(+ f"{item}': '[^']+('\n +'[^']+)*" for item in items+ )+ assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)