Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add LOG_ROOT_FORMATTER to customize the root log handler formatter (#8336)

Adrian707357eebf44
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index b05030a2e..bbe443c3f 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -164,22 +164,7 @@ the settings below.
Logging settings
----------------
-These settings can be used to configure the logging:
-
-* :setting:`LOG_FILE`
-* :setting:`LOG_FILE_APPEND`
-* :setting:`LOG_COLOR`
-* :setting:`LOG_ENABLED`
-* :setting:`LOG_ENCODING`
-* :setting:`LOG_LEVEL`
-* :setting:`LOG_FORMAT`
-* :setting:`LOG_DATEFORMAT`
-* :setting:`LOG_STDOUT`
-* :setting:`LOG_SHORT_NAMES`
-* :setting:`LOG_INSTALL_ROOT_HANDLER`
-
-The first couple of settings define a destination for log messages. If
-:setting:`LOG_FILE` is set, messages sent through the root logger will be
+If :setting:`LOG_FILE` is set, messages sent through the root logger will be
redirected to a file named :setting:`LOG_FILE` with encoding
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
@@ -195,12 +180,6 @@ logger yourself.
messages with lower severity will be filtered out. It ranges through the
possible levels listed in :ref:`topics-logging-levels`.
-:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
-used as layouts for all messages. Those strings can contain any placeholders
-listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
-:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
-respectively.
-
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
Scrapy component responsible for that log output.
@@ -208,6 +187,27 @@ Scrapy component responsible for that log output.
When logging to a terminal, :setting:`LOG_COLOR` colorizes messages by log
level.
+With the default :setting:`LOG_ROOT_FORMATTER`, :setting:`LOG_FORMAT` and
+:setting:`LOG_DATEFORMAT` specify formatting strings used as layouts for all
+messages. Those strings can contain any placeholders listed in :ref:`logging's
+logrecord attributes docs <logrecord-attributes>` and :ref:`datetime's strftime
+and strptime directives <strftime-strptime-behavior>` respectively.
+
+LOG_ROOT_FORMATTER
+~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+.. setting:: LOG_ROOT_FORMATTER
+
+Default: :class:`~scrapy.LogRootFormatter`
+
+:ref:`Formatter <formatter-objects>` of the root log handler that
+:func:`~scrapy.utils.log.configure_logging` installs. It implements
+:class:`~scrapy.LogRootFormatterProtocol`.
+
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
Rotating log files
------------------
@@ -224,6 +224,7 @@ For example, to rotate the log file every day:
import logging
from logging.handlers import TimedRotatingFileHandler
+ from scrapy import LogRootFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
@@ -239,9 +240,7 @@ For example, to rotate the log file every day:
backupCount=7,
encoding=settings.get("LOG_ENCODING"),
)
- handler.setFormatter(
- logging.Formatter(settings.get("LOG_FORMAT"), settings.get("LOG_DATEFORMAT"))
- )
+ handler.setFormatter(LogRootFormatter(settings))
root_logger = logging.getLogger()
root_logger.setLevel(settings.get("LOG_LEVEL"))
@@ -278,10 +277,6 @@ A custom log format can be set for different actions by extending
:class:`~scrapy.logformatter.LogFormatter` class and making
:setting:`LOG_FORMATTER` point to your new class.
-.. autoclass:: scrapy.logformatter.LogFormatter
- :members:
-
-
.. _topics-logging-advanced-customization:
Advanced customization
@@ -402,3 +397,14 @@ scrapy.utils.log module
Refer to :ref:`run-from-script` for more details about using Scrapy this
way.
+
+API reference
+=============
+
+.. autoclass:: scrapy.LogRootFormatterProtocol
+ :members: __init__
+
+.. autoclass:: scrapy.LogRootFormatter
+
+.. autoclass:: scrapy.logformatter.LogFormatter
+ :members:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f512c722a..f5613ad77 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -422,6 +422,7 @@ These settings are:
- :setting:`LOG_INSTALL_ROOT_HANDLER`
- :setting:`LOG_LEVEL`
- :setting:`LOG_LEVELS`
+- :setting:`LOG_ROOT_FORMATTER`
- :setting:`LOG_SHORT_NAMES`
- :setting:`LOG_STDOUT`
@@ -1773,35 +1774,6 @@ overwritten (discarding the output from previous runs, if any).
.. note:: This is a :ref:`logging setting <logging-settings>`.
-.. setting:: LOG_FORMAT
-
-LOG_FORMAT
-----------
-
-Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
-
-String for formatting log messages. Refer to the
-:ref:`Python logging documentation <logrecord-attributes>` for the whole
-list of available placeholders, plus ``%(spider)s`` for the name of the
-spider that triggered the log message, or ``"-"`` for messages not tied to
-a spider (see :ref:`topics-logging-from-spiders`).
-
-.. note:: This is a :ref:`logging setting <logging-settings>`.
-
-.. setting:: LOG_DATEFORMAT
-
-LOG_DATEFORMAT
---------------
-
-Default: ``'%Y-%m-%d %H:%M:%S'``
-
-String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
-in :setting:`LOG_FORMAT`. Refer to the
-:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
-whole list of available directives.
-
-.. note:: This is a :ref:`logging setting <logging-settings>`.
-
.. setting:: LOG_FORMATTER
LOG_FORMATTER
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index d8d40a8f2..77fb71b33 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -7,6 +7,7 @@ import warnings
from importlib.resources import files
# Declare top-level shortcuts
+from scrapy._logging import LogRootFormatter, LogRootFormatterProtocol
from scrapy.http import FormRequest, Request
from scrapy.item import Field, Item
from scrapy.selector import Selector
@@ -16,6 +17,8 @@ __all__ = [
"Field",
"FormRequest",
"Item",
+ "LogRootFormatter",
+ "LogRootFormatterProtocol",
"Request",
"Selector",
"Spider",
diff --git a/scrapy/_logging/__init__.py b/scrapy/_logging/__init__.py
new file mode 100644
index 000000000..da6c0de9a
--- /dev/null
+++ b/scrapy/_logging/__init__.py
@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from logging import Formatter
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+ from scrapy.settings import Settings
+
+
+class LogRootFormatterProtocol(Protocol):
+ """Protocol for log root formatters.
+
+ Besides the ``__init__`` method below, classes implementing this protocol
+ must either subclass :class:`~logging.Formatter` or implement its API.
+
+ Mind that, during a crawl, Scrapy usually creates 2 instances of the log
+ root formatter:
+
+ 1. One early on with only the project *settings*.
+
+ 2. Another one later on, where *settings* also includes add-on and spider
+ settings.
+
+ If you are using multiple :class:`~scrapy.crawler.Crawler` objects, 1
+ log root formatter instance will be created for each crawler.
+ """
+
+ def __init__(self, settings: Settings) -> None: ...
+
+
+class LogRootFormatter(Formatter):
+ """Default :setting:`LOG_ROOT_FORMATTER`.
+
+ Allows customizing log formatting with the following settings.
+
+ .. note:: Alternative log root formatters are encouraged, but not required,
+ to support these settings.
+
+ .. setting:: LOG_FORMAT
+
+ LOG_FORMAT
+ ----------
+
+ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
+
+ String for formatting log messages. Refer to the :ref:`Python logging
+ documentation <logrecord-attributes>` for the whole list of available
+ placeholders, plus ``%(spider)s`` for the name of the spider that
+ triggered the log message, or ``"-"`` for messages not tied to a spider
+ (see :ref:`topics-logging-from-spiders`).
+
+ .. note:: This is a :ref:`logging setting <logging-settings>`.
+
+ .. setting:: LOG_DATEFORMAT
+
+ LOG_DATEFORMAT
+ --------------
+
+ Default: ``'%Y-%m-%d %H:%M:%S'``
+
+ String for formatting date/time, expansion of the ``%(asctime)s``
+ placeholder in :setting:`LOG_FORMAT`. Refer to the :ref:`Python datetime
+ documentation <strftime-strptime-behavior>` for the whole list of available
+ directives.
+
+ .. note:: This is a :ref:`logging setting <logging-settings>`.
+ """
+
+ def __init__(self, settings: Settings):
+ super().__init__(fmt=settings["LOG_FORMAT"], datefmt=settings["LOG_DATEFORMAT"])
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 479b6cc16..25fbf6cf6 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -156,6 +156,7 @@ __all__ = [
"LOG_INSTALL_ROOT_HANDLER",
"LOG_LEVEL",
"LOG_LEVELS",
+ "LOG_ROOT_FORMATTER",
"LOG_SHORT_NAMES",
"LOG_STDOUT",
"LOG_VERSIONS",
@@ -487,6 +488,7 @@ LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
LOG_LEVEL = "DEBUG"
LOG_LEVELS: dict[str, str] = {}
+LOG_ROOT_FORMATTER = "scrapy.LogRootFormatter"
LOG_SHORT_NAMES = False
LOG_STDOUT = False
LOG_VERSIONS = [
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 8c46bdba5..4fa8a16ae 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -18,6 +18,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils._colorize import tty_supports_color
from scrapy.utils._versions import get_versions
+from scrapy.utils.misc import load_object
if TYPE_CHECKING:
from types import TracebackType
@@ -200,9 +201,32 @@ def get_scrapy_root_handler() -> logging.Handler | None:
return _scrapy_root_handler
+class _ColoredFormatter(logging.Formatter):
+ """Colorizes the output of *formatter* by log level with *colorizer*, a
+ :class:`colorlog.ColoredFormatter` that only renders the message.
+ """
+
+ def __init__(self, formatter: logging.Formatter, colorizer: logging.Formatter):
+ super().__init__()
+ self._formatter = formatter
+ self._colorizer = colorizer
+
+ def format(self, record: logging.LogRecord) -> str:
+ formatted = logging.makeLogRecord(
+ {
+ **record.__dict__,
+ "msg": self._formatter.format(record),
+ "args": None,
+ "exc_info": None,
+ "exc_text": None,
+ "stack_info": None,
+ }
+ )
+ return self._colorizer.format(formatted)
+
+
def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Formatter:
- fmt = settings.get("LOG_FORMAT")
- datefmt = settings.get("LOG_DATEFORMAT")
+ formatter: logging.Formatter = load_object(settings["LOG_ROOT_FORMATTER"])(settings)
if (
isinstance(handler, logging.StreamHandler)
and not isinstance(handler, logging.FileHandler)
@@ -215,8 +239,9 @@ def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Form
except ImportError:
pass
else:
- return ColoredFormatter(fmt=f"%(log_color)s{fmt}", datefmt=datefmt)
- return logging.Formatter(fmt=fmt, datefmt=datefmt)
+ colorizer = ColoredFormatter("%(log_color)s%(message)s")
+ return _ColoredFormatter(formatter, colorizer)
+ return formatter
def _get_handler(settings: Settings) -> logging.Handler:
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index d3a14876e..51dcdef2b 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -11,6 +11,7 @@ import pytest
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
+from scrapy import LogRootFormatter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.log import (
@@ -176,6 +177,14 @@ class _TTYStringIO(StringIO):
return True
+class _UpperFormatter(logging.Formatter):
+ def __init__(self, settings: Settings):
+ super().__init__(settings["LOG_FORMAT"])
+
+ def format(self, record: logging.LogRecord) -> str:
+ return super().format(record).upper()
+
+
class TestGetFormatter:
@staticmethod
def _settings(**overrides: Any) -> Settings:
@@ -194,22 +203,33 @@ class TestGetFormatter:
monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)
stream = _TTYStringIO()
handler = logging.StreamHandler(stream)
- handler.setFormatter(_get_formatter(handler, self._settings()))
+ settings = self._settings(LOG_ROOT_FORMATTER=_UpperFormatter)
+ handler.setFormatter(_get_formatter(handler, settings))
logger = logging.getLogger("test_get_formatter_colored")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
try:
- logger.error("boom")
+ try:
+ raise ValueError("bar")
+ except ValueError:
+ logger.exception("boom")
finally:
logger.removeHandler(handler)
output = stream.getvalue()
assert "\x1b[" in output
- assert "boom" in output
+ assert "ERROR: BOOM" in output
+ assert output.count("VALUEERROR: BAR") == 1
def test_plain_when_not_a_tty(self) -> None:
handler = logging.StreamHandler(StringIO())
formatter = _get_formatter(handler, self._settings())
- assert type(formatter) is logging.Formatter
+ assert type(formatter) is LogRootFormatter
+
+ def test_custom_root_formatter(self) -> None:
+ handler = logging.StreamHandler(StringIO())
+ settings = self._settings(LOG_ROOT_FORMATTER=_UpperFormatter)
+ formatter = _get_formatter(handler, settings)
+ assert type(formatter) is _UpperFormatter
def test_plain_when_log_color_disabled(
self, monkeypatch: pytest.MonkeyPatch
@@ -217,7 +237,7 @@ class TestGetFormatter:
monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)
handler = logging.StreamHandler(_TTYStringIO())
formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))
- assert type(formatter) is logging.Formatter
+ assert type(formatter) is LogRootFormatter
def test_plain_for_file_handler(self, tmp_path: Path) -> None:
handler = logging.FileHandler(tmp_path / "log.txt")
@@ -225,7 +245,7 @@ class TestGetFormatter:
formatter = _get_formatter(handler, self._settings())
finally:
handler.close()
- assert type(formatter) is logging.Formatter
+ assert type(formatter) is LogRootFormatter
def test_plain_when_colorlog_not_installed(
self, monkeypatch: pytest.MonkeyPatch
@@ -234,7 +254,7 @@ class TestGetFormatter:
monkeypatch.setitem(sys.modules, "colorlog", None)
handler = logging.StreamHandler(_TTYStringIO())
formatter = _get_formatter(handler, self._settings())
- assert type(formatter) is logging.Formatter
+ assert type(formatter) is LogRootFormatter
class TestConfigureLogging: