Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add LOG_INSTALL_ROOT_HANDLER setting and deprecate the install_root_handler parameter (#8041)

Adrianf9ade922a732
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index bf1a0bec7..8ac691840 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -166,6 +166,7 @@ These settings can be used to configure the logging:
* :setting:`LOG_DATEFORMAT`
* :setting:`LOG_STDOUT`
* :setting:`LOG_SHORT_NAMES`
+* :setting:`LOG_INSTALL_ROOT_HANDLER`
The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
@@ -176,6 +177,10 @@ and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
(discarding the output from previous runs, if any). Lastly, if
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
+Set :setting:`LOG_INSTALL_ROOT_HANDLER` to ``False`` to prevent Scrapy from
+installing a root logging handler altogether, e.g. to configure the root
+logger yourself.
+
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
messages with lower severity will be filtered out. It ranges through the
possible levels listed in :ref:`topics-logging-levels`.
@@ -215,7 +220,8 @@ For example, to rotate the log file every day:
from myproject.spiders.myspider import MySpider
settings = get_project_settings()
- process = CrawlerProcess(settings, install_root_handler=False)
+ settings.set("LOG_INSTALL_ROOT_HANDLER", False)
+ process = CrawlerProcess(settings)
handler = TimedRotatingFileHandler(
"scrapy.log",
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cacc161ab..77e8585be 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -408,6 +408,7 @@ These settings are:
- :setting:`LOG_FILE`
- :setting:`LOG_FILE_APPEND`
- :setting:`LOG_FORMAT`
+- :setting:`LOG_INSTALL_ROOT_HANDLER`
- :setting:`LOG_LEVEL`
- :setting:`LOG_SHORT_NAMES`
- :setting:`LOG_STDOUT`
@@ -1710,6 +1711,20 @@ Default: :class:`scrapy.logformatter.LogFormatter`
The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
+.. setting:: LOG_INSTALL_ROOT_HANDLER
+
+LOG_INSTALL_ROOT_HANDLER
+-------------------------
+
+Default: ``True``
+
+Whether to install a handler for the root logger, configured according to the
+other :ref:`logging settings <logging-settings>`. Set this to ``False`` to
+manage the root logger yourself, e.g. from a custom command or from a
+:file:`settings.py` module executed before Scrapy configures logging.
+
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
.. setting:: LOG_LEVEL
LOG_LEVEL
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 304b63d37..554cccc6e 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -718,7 +718,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
def __init__(
self,
settings: dict[str, Any] | Settings | None = None,
- install_root_handler: bool = True,
+ install_root_handler: bool | None = None,
):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
@@ -838,7 +838,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
:class:`~scrapy.settings.Settings` object.
:param install_root_handler: whether to install root logging handler
- (default: True)
+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting)
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
@@ -851,7 +851,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
def __init__(
self,
settings: dict[str, Any] | Settings | None = None,
- install_root_handler: bool = True,
+ install_root_handler: bool | None = None,
):
super().__init__(settings, install_root_handler)
self._initialized_reactor: bool = False
@@ -918,7 +918,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
to not be installed but installs an asyncio event loop and uses it.
:param install_root_handler: whether to install root logging handler
- (default: True)
+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting)
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
@@ -932,7 +932,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
def __init__(
self,
settings: dict[str, Any] | Settings | None = None,
- install_root_handler: bool = True,
+ install_root_handler: bool | None = None,
):
super().__init__(settings, install_root_handler)
logger.debug("Using AsyncCrawlerProcess")
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e8157f538..31e326016 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,6 +146,7 @@ __all__ = [
"LOG_FILE_APPEND",
"LOG_FORMAT",
"LOG_FORMATTER",
+ "LOG_INSTALL_ROOT_HANDLER",
"LOG_LEVEL",
"LOG_SHORT_NAMES",
"LOG_STDOUT",
@@ -453,6 +454,7 @@ JOBDIR = None
LOG_COLOR = True
LOG_ENABLED = True
+LOG_INSTALL_ROOT_HANDLER = True
LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"
LOG_ENCODING = "utf-8"
LOG_FILE = None
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 94e1cd5f1..9d0e95da3 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -97,7 +97,7 @@ DEFAULT_LOGGING = {
def configure_logging(
settings: Settings | dict[str, Any] | None = None,
- install_root_handler: bool = True,
+ install_root_handler: bool | None = None,
) -> None:
"""
Initialize logging defaults for Scrapy.
@@ -107,7 +107,7 @@ def configure_logging(
:type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
:param install_root_handler: whether to install root logging handler
- (default: True)
+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting)
:type install_root_handler: bool
This function does:
@@ -116,12 +116,25 @@ def configure_logging(
- Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
- Route stdout to log if LOG_STDOUT setting is True
- When ``install_root_handler`` is True (default), this function also
- creates a handler for the root logger according to given settings
- (see :ref:`topics-logging-settings`). You can override default options
- using ``settings`` argument. When ``settings`` is empty or None, defaults
- are used.
+ When installing a root logging handler, this function also creates a
+ handler for the root logger according to given settings (see
+ :ref:`topics-logging-settings`). You can override default options using
+ ``settings`` argument. When ``settings`` is empty or None, defaults are
+ used.
"""
+ if isinstance(settings, dict) or settings is None:
+ settings = Settings(settings)
+
+ if install_root_handler is None:
+ install_root_handler = settings.getbool("LOG_INSTALL_ROOT_HANDLER")
+ else:
+ warnings.warn(
+ "The install_root_handler parameter is deprecated. Set the "
+ "LOG_INSTALL_ROOT_HANDLER setting instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
if not sys.warnoptions:
# Route warnings through python logging
logging.captureWarnings(True)
@@ -131,9 +144,6 @@ def configure_logging(
dictConfig(DEFAULT_LOGGING)
- if isinstance(settings, dict) or settings is None:
- settings = Settings(settings)
-
if settings.getbool("LOG_STDOUT"):
sys.stdout = StreamLogger(logging.getLogger("stdout"))
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8b3e8b7c6..bb5c20f18 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -502,6 +502,48 @@ class MySpider(scrapy.Spider):
self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")
+class TestLogInstallRootHandler(TestProjectBase):
+ """LOG_INSTALL_ROOT_HANDLER=False must let a user-configured root handler
+ be the only one in effect, instead of Scrapy adding its own alongside it."""
+
+ @pytest.fixture(autouse=True)
+ def create_files(self, proj_path: Path) -> None:
+ proj_mod_path = proj_path / self.project_name
+ (proj_mod_path / "spiders" / "sp.py").write_text("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'sp'
+
+ async def start(self):
+ self.logger.info('It works!')
+ return
+ yield
+""")
+ self._append_settings(
+ proj_mod_path,
+ "\nimport logging\n"
+ "logging.basicConfig(level=logging.INFO, format='CUSTOM: %(message)s')\n",
+ )
+
+ def test_default(self, proj_path: Path) -> None:
+ """Scrapy installs its own root handler in addition to the
+ user-configured one, so the message is logged twice, once in each
+ format."""
+ _, _, err = proc("crawl", "sp", cwd=proj_path)
+ assert "CUSTOM: It works!" in err
+ assert "[sp] INFO: It works!" in err
+
+ def test_disabled(self, proj_path: Path) -> None:
+ """Scrapy leaves the user-configured root handler alone, so the
+ message is only logged once, in the user's format."""
+ _, _, err = proc(
+ "crawl", "sp", "-s", "LOG_INSTALL_ROOT_HANDLER=False", cwd=proj_path
+ )
+ assert "CUSTOM: It works!" in err
+ assert "[sp] INFO: It works!" not in err
+
+
class TestMiscCommands(TestProjectBase):
def test_list(self, proj_path: Path) -> None:
assert call("list", cwd=proj_path) == 0
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 9e22839a1..9bdaa6f38 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -858,7 +858,7 @@ def test_log_scrapy_info(
settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level("INFO"):
- CrawlerProcess(settings, install_root_handler=False)
+ CrawlerProcess({**settings, "LOG_INSTALL_ROOT_HANDLER": False})
assert (
caplog.records[0].getMessage()
== f"Scrapy {scrapy.__version__} started (bot: scrapybot)"
diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py
index 9e35a38e2..0b3a52cfe 100644
--- a/tests/test_crawler_runners.py
+++ b/tests/test_crawler_runners.py
@@ -13,6 +13,7 @@ from scrapy.crawler import (
CrawlerRunner,
CrawlerRunnerBase,
)
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_reactor_settings
@@ -54,12 +55,13 @@ class TestAsyncCrawlerRunner:
class TestCrawlerProcess:
def test_crawler_process_accepts_dict(self) -> None:
- runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
+ runner = CrawlerProcess({"foo": "bar", "LOG_INSTALL_ROOT_HANDLER": False})
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
def test_crawler_process_accepts_None(self) -> None:
- runner = CrawlerProcess(install_root_handler=False)
+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):
+ runner = CrawlerProcess(install_root_handler=False)
assert_option_is_default(runner.settings, "RETRY_ENABLED")
@@ -67,15 +69,19 @@ class TestCrawlerProcess:
class TestAsyncCrawlerProcess:
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
runner = AsyncCrawlerProcess(
- {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
- install_root_handler=False,
+ {
+ "foo": "bar",
+ "TWISTED_REACTOR_ENABLED": reactor_pytest != "none",
+ "LOG_INSTALL_ROOT_HANDLER": False,
+ }
)
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
def test_crawler_process_accepts_None(self) -> None:
- runner = AsyncCrawlerProcess(install_root_handler=False)
+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):
+ runner = AsyncCrawlerProcess(install_root_handler=False)
assert_option_is_default(runner.settings, "RETRY_ENABLED")
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index f4cfd2223..ce8da0b39 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -232,21 +232,27 @@ class TestConfigureLogging:
return warnings.showwarning.__module__ == "logging"
def test_log_stdout(self) -> None:
- configure_logging(settings={"LOG_STDOUT": True}, install_root_handler=False)
+ configure_logging(
+ settings={"LOG_STDOUT": True, "LOG_INSTALL_ROOT_HANDLER": False}
+ )
assert isinstance(sys.stdout, StreamLogger)
def test_captures_warnings(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sys, "warnoptions", [])
logging.captureWarnings(False)
- configure_logging(install_root_handler=False)
+ configure_logging(settings={"LOG_INSTALL_ROOT_HANDLER": False})
assert self._warnings_are_captured()
def test_keeps_warnoptions(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sys, "warnoptions", ["default"])
logging.captureWarnings(False)
- configure_logging(install_root_handler=False)
+ configure_logging(settings={"LOG_INSTALL_ROOT_HANDLER": False})
assert not self._warnings_are_captured()
+ def test_install_root_handler_param_deprecated(self) -> None:
+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):
+ configure_logging(install_root_handler=False)
+
def test_reinstall_root_handler_removed_from_root(self) -> None:
install_scrapy_root_handler(Settings())
handler = get_scrapy_root_handler()
diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py
index 496c65edd..aa6e98e7d 100644
--- a/tests/test_zz_resources.py
+++ b/tests/test_zz_resources.py
@@ -26,8 +26,9 @@ def test_stderr_log_handler() -> None:
"""Test that the Scrapy root handler is always properly removed.
It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess``
- (without ``install_root_handler=False``). It can be removed with
- ``_uninstall_scrapy_root_handler()`` if installing it was really needed.
+ (unless the ``LOG_INSTALL_ROOT_HANDLER`` setting is ``False``). It can be
+ removed with ``_uninstall_scrapy_root_handler()`` if installing it was
+ really needed.
"""
c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler)
assert c == 0