Scrapy, a fast high-level web crawling & scraping framework for Python.
Add LOG_ROOT_FORMATTER to customize the root log handler formatter (#8336)
Adrian707357eebf44
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rstindex b05030a2e..bbe443c3f 100644--- a/docs/topics/logging.rst+++ b/docs/topics/logging.rst@@ -164,22 +164,7 @@ the settings below.Logging settings-----------------These settings can be used to configure the logging:--* :setting:`LOG_FILE`-* :setting:`LOG_FILE_APPEND`-* :setting:`LOG_COLOR`-* :setting:`LOG_ENABLED`-* :setting:`LOG_ENCODING`-* :setting:`LOG_LEVEL`-* :setting:`LOG_FORMAT`-* :setting:`LOG_DATEFORMAT`-* :setting:`LOG_STDOUT`-* :setting:`LOG_SHORT_NAMES`-* :setting:`LOG_INSTALL_ROOT_HANDLER`--The first couple of settings define a destination for log messages. If-:setting:`LOG_FILE` is set, messages sent through the root logger will be+If :setting:`LOG_FILE` is set, messages sent through the root logger will beredirected to a file named :setting:`LOG_FILE` with encoding:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, logmessages will be displayed on the standard error. If :setting:`LOG_FILE` is set@@ -195,12 +180,6 @@ logger yourself.messages with lower severity will be filtered out. It ranges through thepossible levels listed in :ref:`topics-logging-levels`.-:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings-used as layouts for all messages. Those strings can contain any placeholders-listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and-:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`-respectively.-If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapycomponent that prints the log. It is unset by default, hence logs contain theScrapy component responsible for that log output.@@ -208,6 +187,27 @@ Scrapy component responsible for that log output.When logging to a terminal, :setting:`LOG_COLOR` colorizes messages by loglevel.+With the default :setting:`LOG_ROOT_FORMATTER`, :setting:`LOG_FORMAT` and+:setting:`LOG_DATEFORMAT` specify formatting strings used as layouts for all+messages. Those strings can contain any placeholders listed in :ref:`logging's+logrecord attributes docs <logrecord-attributes>` and :ref:`datetime's strftime+and strptime directives <strftime-strptime-behavior>` respectively.++LOG_ROOT_FORMATTER+~~~~~~~~~~~~~~~~~~++.. versionadded:: VERSION++.. setting:: LOG_ROOT_FORMATTER++Default: :class:`~scrapy.LogRootFormatter`++:ref:`Formatter <formatter-objects>` of the root log handler that+:func:`~scrapy.utils.log.configure_logging` installs. It implements+:class:`~scrapy.LogRootFormatterProtocol`.++.. note:: This is a :ref:`logging setting <logging-settings>`.+Rotating log files------------------@@ -224,6 +224,7 @@ For example, to rotate the log file every day:import loggingfrom logging.handlers import TimedRotatingFileHandler+ from scrapy import LogRootFormatterfrom scrapy.crawler import CrawlerProcessfrom scrapy.utils.project import get_project_settings@@ -239,9 +240,7 @@ For example, to rotate the log file every day:backupCount=7,encoding=settings.get("LOG_ENCODING"),)- handler.setFormatter(- logging.Formatter(settings.get("LOG_FORMAT"), settings.get("LOG_DATEFORMAT"))- )+ handler.setFormatter(LogRootFormatter(settings))root_logger = logging.getLogger()root_logger.setLevel(settings.get("LOG_LEVEL"))@@ -278,10 +277,6 @@ A custom log format can be set for different actions by extending:class:`~scrapy.logformatter.LogFormatter` class and making:setting:`LOG_FORMATTER` point to your new class.-.. autoclass:: scrapy.logformatter.LogFormatter- :members:--.. _topics-logging-advanced-customization:Advanced customization@@ -402,3 +397,14 @@ scrapy.utils.log moduleRefer to :ref:`run-from-script` for more details about using Scrapy thisway.++API reference+=============++.. autoclass:: scrapy.LogRootFormatterProtocol+ :members: __init__++.. autoclass:: scrapy.LogRootFormatter++.. autoclass:: scrapy.logformatter.LogFormatter+ :members:diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex f512c722a..f5613ad77 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -422,6 +422,7 @@ These settings are:- :setting:`LOG_INSTALL_ROOT_HANDLER`- :setting:`LOG_LEVEL`- :setting:`LOG_LEVELS`+- :setting:`LOG_ROOT_FORMATTER`- :setting:`LOG_SHORT_NAMES`- :setting:`LOG_STDOUT`@@ -1773,35 +1774,6 @@ overwritten (discarding the output from previous runs, if any)... note:: This is a :ref:`logging setting <logging-settings>`.-.. setting:: LOG_FORMAT--LOG_FORMAT-------------Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``--String for formatting log messages. Refer to the-:ref:`Python logging documentation <logrecord-attributes>` for the whole-list of available placeholders, plus ``%(spider)s`` for the name of the-spider that triggered the log message, or ``"-"`` for messages not tied to-a spider (see :ref:`topics-logging-from-spiders`).--.. note:: This is a :ref:`logging setting <logging-settings>`.--.. setting:: LOG_DATEFORMAT--LOG_DATEFORMAT-----------------Default: ``'%Y-%m-%d %H:%M:%S'``--String for formatting date/time, expansion of the ``%(asctime)s`` placeholder-in :setting:`LOG_FORMAT`. Refer to the-:ref:`Python datetime documentation <strftime-strptime-behavior>` for the-whole list of available directives.--.. note:: This is a :ref:`logging setting <logging-settings>`.-.. setting:: LOG_FORMATTERLOG_FORMATTERdiff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex d8d40a8f2..77fb71b33 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -7,6 +7,7 @@ import warningsfrom importlib.resources import files# Declare top-level shortcuts+from scrapy._logging import LogRootFormatter, LogRootFormatterProtocolfrom scrapy.http import FormRequest, Requestfrom scrapy.item import Field, Itemfrom scrapy.selector import Selector@@ -16,6 +17,8 @@ __all__ = ["Field","FormRequest","Item",+ "LogRootFormatter",+ "LogRootFormatterProtocol","Request","Selector","Spider",diff --git a/scrapy/_logging/__init__.py b/scrapy/_logging/__init__.pynew file mode 100644index 000000000..da6c0de9a--- /dev/null+++ b/scrapy/_logging/__init__.py@@ -0,0 +1,70 @@+from __future__ import annotations++from logging import Formatter+from typing import TYPE_CHECKING, Protocol++if TYPE_CHECKING:+ from scrapy.settings import Settings+++class LogRootFormatterProtocol(Protocol):+ """Protocol for log root formatters.++ Besides the ``__init__`` method below, classes implementing this protocol+ must either subclass :class:`~logging.Formatter` or implement its API.++ Mind that, during a crawl, Scrapy usually creates 2 instances of the log+ root formatter:++ 1. One early on with only the project *settings*.++ 2. Another one later on, where *settings* also includes add-on and spider+ settings.++ If you are using multiple :class:`~scrapy.crawler.Crawler` objects, 1+ log root formatter instance will be created for each crawler.+ """++ def __init__(self, settings: Settings) -> None: ...+++class LogRootFormatter(Formatter):+ """Default :setting:`LOG_ROOT_FORMATTER`.++ Allows customizing log formatting with the following settings.++ .. note:: Alternative log root formatters are encouraged, but not required,+ to support these settings.++ .. setting:: LOG_FORMAT++ LOG_FORMAT+ ----------++ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``++ String for formatting log messages. Refer to the :ref:`Python logging+ documentation <logrecord-attributes>` for the whole list of available+ placeholders, plus ``%(spider)s`` for the name of the spider that+ triggered the log message, or ``"-"`` for messages not tied to a spider+ (see :ref:`topics-logging-from-spiders`).++ .. note:: This is a :ref:`logging setting <logging-settings>`.++ .. setting:: LOG_DATEFORMAT++ LOG_DATEFORMAT+ --------------++ Default: ``'%Y-%m-%d %H:%M:%S'``++ String for formatting date/time, expansion of the ``%(asctime)s``+ placeholder in :setting:`LOG_FORMAT`. Refer to the :ref:`Python datetime+ documentation <strftime-strptime-behavior>` for the whole list of available+ directives.++ .. note:: This is a :ref:`logging setting <logging-settings>`.+ """++ def __init__(self, settings: Settings):+ super().__init__(fmt=settings["LOG_FORMAT"], datefmt=settings["LOG_DATEFORMAT"])diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 479b6cc16..25fbf6cf6 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -156,6 +156,7 @@ __all__ = ["LOG_INSTALL_ROOT_HANDLER","LOG_LEVEL","LOG_LEVELS",+ "LOG_ROOT_FORMATTER","LOG_SHORT_NAMES","LOG_STDOUT","LOG_VERSIONS",@@ -487,6 +488,7 @@ LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"LOG_FORMATTER = "scrapy.logformatter.LogFormatter"LOG_LEVEL = "DEBUG"LOG_LEVELS: dict[str, str] = {}+LOG_ROOT_FORMATTER = "scrapy.LogRootFormatter"LOG_SHORT_NAMES = FalseLOG_STDOUT = FalseLOG_VERSIONS = [diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 8c46bdba5..4fa8a16ae 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -18,6 +18,7 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settingsfrom scrapy.utils._colorize import tty_supports_colorfrom scrapy.utils._versions import get_versions+from scrapy.utils.misc import load_objectif TYPE_CHECKING:from types import TracebackType@@ -200,9 +201,32 @@ def get_scrapy_root_handler() -> logging.Handler | None:return _scrapy_root_handler+class _ColoredFormatter(logging.Formatter):+ """Colorizes the output of *formatter* by log level with *colorizer*, a+ :class:`colorlog.ColoredFormatter` that only renders the message.+ """++ def __init__(self, formatter: logging.Formatter, colorizer: logging.Formatter):+ super().__init__()+ self._formatter = formatter+ self._colorizer = colorizer++ def format(self, record: logging.LogRecord) -> str:+ formatted = logging.makeLogRecord(+ {+ **record.__dict__,+ "msg": self._formatter.format(record),+ "args": None,+ "exc_info": None,+ "exc_text": None,+ "stack_info": None,+ }+ )+ return self._colorizer.format(formatted)++def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Formatter:- fmt = settings.get("LOG_FORMAT")- datefmt = settings.get("LOG_DATEFORMAT")+ formatter: logging.Formatter = load_object(settings["LOG_ROOT_FORMATTER"])(settings)if (isinstance(handler, logging.StreamHandler)and not isinstance(handler, logging.FileHandler)@@ -215,8 +239,9 @@ def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Formexcept ImportError:passelse:- return ColoredFormatter(fmt=f"%(log_color)s{fmt}", datefmt=datefmt)- return logging.Formatter(fmt=fmt, datefmt=datefmt)+ colorizer = ColoredFormatter("%(log_color)s%(message)s")+ return _ColoredFormatter(formatter, colorizer)+ return formatterdef _get_handler(settings: Settings) -> logging.Handler:diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex d3a14876e..51dcdef2b 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -11,6 +11,7 @@ import pytestfrom twisted.python import log as twisted_logfrom twisted.python.failure import Failure+from scrapy import LogRootFormatterfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settingsfrom scrapy.utils.log import (@@ -176,6 +177,14 @@ class _TTYStringIO(StringIO):return True+class _UpperFormatter(logging.Formatter):+ def __init__(self, settings: Settings):+ super().__init__(settings["LOG_FORMAT"])++ def format(self, record: logging.LogRecord) -> str:+ return super().format(record).upper()++class TestGetFormatter:@staticmethoddef _settings(**overrides: Any) -> Settings:@@ -194,22 +203,33 @@ class TestGetFormatter:monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)stream = _TTYStringIO()handler = logging.StreamHandler(stream)- handler.setFormatter(_get_formatter(handler, self._settings()))+ settings = self._settings(LOG_ROOT_FORMATTER=_UpperFormatter)+ handler.setFormatter(_get_formatter(handler, settings))logger = logging.getLogger("test_get_formatter_colored")logger.addHandler(handler)logger.setLevel(logging.DEBUG)try:- logger.error("boom")+ try:+ raise ValueError("bar")+ except ValueError:+ logger.exception("boom")finally:logger.removeHandler(handler)output = stream.getvalue()assert "\x1b[" in output- assert "boom" in output+ assert "ERROR: BOOM" in output+ assert output.count("VALUEERROR: BAR") == 1def test_plain_when_not_a_tty(self) -> None:handler = logging.StreamHandler(StringIO())formatter = _get_formatter(handler, self._settings())- assert type(formatter) is logging.Formatter+ assert type(formatter) is LogRootFormatter++ def test_custom_root_formatter(self) -> None:+ handler = logging.StreamHandler(StringIO())+ settings = self._settings(LOG_ROOT_FORMATTER=_UpperFormatter)+ formatter = _get_formatter(handler, settings)+ assert type(formatter) is _UpperFormatterdef test_plain_when_log_color_disabled(self, monkeypatch: pytest.MonkeyPatch@@ -217,7 +237,7 @@ class TestGetFormatter:monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)handler = logging.StreamHandler(_TTYStringIO())formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))- assert type(formatter) is logging.Formatter+ assert type(formatter) is LogRootFormatterdef test_plain_for_file_handler(self, tmp_path: Path) -> None:handler = logging.FileHandler(tmp_path / "log.txt")@@ -225,7 +245,7 @@ class TestGetFormatter:formatter = _get_formatter(handler, self._settings())finally:handler.close()- assert type(formatter) is logging.Formatter+ assert type(formatter) is LogRootFormatterdef test_plain_when_colorlog_not_installed(self, monkeypatch: pytest.MonkeyPatch@@ -234,7 +254,7 @@ class TestGetFormatter:monkeypatch.setitem(sys.modules, "colorlog", None)handler = logging.StreamHandler(_TTYStringIO())formatter = _get_formatter(handler, self._settings())- assert type(formatter) is logging.Formatter+ assert type(formatter) is LogRootFormatterclass TestConfigureLogging: