Scrapy, a fast high-level web crawling & scraping framework for Python.
Add LOG_INSTALL_ROOT_HANDLER setting and deprecate the install_root_handler parameter (#8041)
Adrianf9ade922a732
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rstindex bf1a0bec7..8ac691840 100644--- a/docs/topics/logging.rst+++ b/docs/topics/logging.rst@@ -166,6 +166,7 @@ These settings can be used to configure the logging:* :setting:`LOG_DATEFORMAT`* :setting:`LOG_STDOUT`* :setting:`LOG_SHORT_NAMES`+* :setting:`LOG_INSTALL_ROOT_HANDLER`The first couple of settings define a destination for log messages. If:setting:`LOG_FILE` is set, messages sent through the root logger will be@@ -176,6 +177,10 @@ and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten(discarding the output from previous runs, if any). Lastly, if:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.+Set :setting:`LOG_INSTALL_ROOT_HANDLER` to ``False`` to prevent Scrapy from+installing a root logging handler altogether, e.g. to configure the root+logger yourself.+:setting:`LOG_LEVEL` determines the minimum level of severity to display, thosemessages with lower severity will be filtered out. It ranges through thepossible levels listed in :ref:`topics-logging-levels`.@@ -215,7 +220,8 @@ For example, to rotate the log file every day:from myproject.spiders.myspider import MySpidersettings = get_project_settings()- process = CrawlerProcess(settings, install_root_handler=False)+ settings.set("LOG_INSTALL_ROOT_HANDLER", False)+ process = CrawlerProcess(settings)handler = TimedRotatingFileHandler("scrapy.log",diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex cacc161ab..77e8585be 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -408,6 +408,7 @@ These settings are:- :setting:`LOG_FILE`- :setting:`LOG_FILE_APPEND`- :setting:`LOG_FORMAT`+- :setting:`LOG_INSTALL_ROOT_HANDLER`- :setting:`LOG_LEVEL`- :setting:`LOG_SHORT_NAMES`- :setting:`LOG_STDOUT`@@ -1710,6 +1711,20 @@ Default: :class:`scrapy.logformatter.LogFormatter`The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.+.. setting:: LOG_INSTALL_ROOT_HANDLER++LOG_INSTALL_ROOT_HANDLER+-------------------------++Default: ``True``++Whether to install a handler for the root logger, configured according to the+other :ref:`logging settings <logging-settings>`. Set this to ``False`` to+manage the root logger yourself, e.g. from a custom command or from a+:file:`settings.py` module executed before Scrapy configures logging.++.. note:: This is a :ref:`logging setting <logging-settings>`.+.. setting:: LOG_LEVELLOG_LEVELdiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 304b63d37..554cccc6e 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -718,7 +718,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):def __init__(self,settings: dict[str, Any] | Settings | None = None,- install_root_handler: bool = True,+ install_root_handler: bool | None = None,):super().__init__(settings)configure_logging(self.settings, install_root_handler)@@ -838,7 +838,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner)::class:`~scrapy.settings.Settings` object.:param install_root_handler: whether to install root logging handler- (default: True)+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting)This class shouldn't be needed (since Scrapy is responsible of using itaccordingly) unless writing scripts that manually handle the crawling@@ -851,7 +851,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):def __init__(self,settings: dict[str, Any] | Settings | None = None,- install_root_handler: bool = True,+ install_root_handler: bool | None = None,):super().__init__(settings, install_root_handler)self._initialized_reactor: bool = False@@ -918,7 +918,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):to not be installed but installs an asyncio event loop and uses it.:param install_root_handler: whether to install root logging handler- (default: True)+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting)This class shouldn't be needed (since Scrapy is responsible of using itaccordingly) unless writing scripts that manually handle the crawling@@ -932,7 +932,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):def __init__(self,settings: dict[str, Any] | Settings | None = None,- install_root_handler: bool = True,+ install_root_handler: bool | None = None,):super().__init__(settings, install_root_handler)logger.debug("Using AsyncCrawlerProcess")diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex e8157f538..31e326016 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -146,6 +146,7 @@ __all__ = ["LOG_FILE_APPEND","LOG_FORMAT","LOG_FORMATTER",+ "LOG_INSTALL_ROOT_HANDLER","LOG_LEVEL","LOG_SHORT_NAMES","LOG_STDOUT",@@ -453,6 +454,7 @@ JOBDIR = NoneLOG_COLOR = TrueLOG_ENABLED = True+LOG_INSTALL_ROOT_HANDLER = TrueLOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"LOG_ENCODING = "utf-8"LOG_FILE = Nonediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 94e1cd5f1..9d0e95da3 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -97,7 +97,7 @@ DEFAULT_LOGGING = {def configure_logging(settings: Settings | dict[str, Any] | None = None,- install_root_handler: bool = True,+ install_root_handler: bool | None = None,) -> None:"""Initialize logging defaults for Scrapy.@@ -107,7 +107,7 @@ def configure_logging(:type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``:param install_root_handler: whether to install root logging handler- (default: True)+ (default: the :setting:`LOG_INSTALL_ROOT_HANDLER` setting):type install_root_handler: boolThis function does:@@ -116,12 +116,25 @@ def configure_logging(- Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively- Route stdout to log if LOG_STDOUT setting is True- When ``install_root_handler`` is True (default), this function also- creates a handler for the root logger according to given settings- (see :ref:`topics-logging-settings`). You can override default options- using ``settings`` argument. When ``settings`` is empty or None, defaults- are used.+ When installing a root logging handler, this function also creates a+ handler for the root logger according to given settings (see+ :ref:`topics-logging-settings`). You can override default options using+ ``settings`` argument. When ``settings`` is empty or None, defaults are+ used."""+ if isinstance(settings, dict) or settings is None:+ settings = Settings(settings)++ if install_root_handler is None:+ install_root_handler = settings.getbool("LOG_INSTALL_ROOT_HANDLER")+ else:+ warnings.warn(+ "The install_root_handler parameter is deprecated. Set the "+ "LOG_INSTALL_ROOT_HANDLER setting instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+if not sys.warnoptions:# Route warnings through python logginglogging.captureWarnings(True)@@ -131,9 +144,6 @@ def configure_logging(dictConfig(DEFAULT_LOGGING)- if isinstance(settings, dict) or settings is None:- settings = Settings(settings)-if settings.getbool("LOG_STDOUT"):sys.stdout = StreamLogger(logging.getLogger("stdout"))diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 8b3e8b7c6..bb5c20f18 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -502,6 +502,48 @@ class MySpider(scrapy.Spider):self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")+class TestLogInstallRootHandler(TestProjectBase):+ """LOG_INSTALL_ROOT_HANDLER=False must let a user-configured root handler+ be the only one in effect, instead of Scrapy adding its own alongside it."""++ @pytest.fixture(autouse=True)+ def create_files(self, proj_path: Path) -> None:+ proj_mod_path = proj_path / self.project_name+ (proj_mod_path / "spiders" / "sp.py").write_text("""+import scrapy++class MySpider(scrapy.Spider):+ name = 'sp'++ async def start(self):+ self.logger.info('It works!')+ return+ yield+""")+ self._append_settings(+ proj_mod_path,+ "\nimport logging\n"+ "logging.basicConfig(level=logging.INFO, format='CUSTOM: %(message)s')\n",+ )++ def test_default(self, proj_path: Path) -> None:+ """Scrapy installs its own root handler in addition to the+ user-configured one, so the message is logged twice, once in each+ format."""+ _, _, err = proc("crawl", "sp", cwd=proj_path)+ assert "CUSTOM: It works!" in err+ assert "[sp] INFO: It works!" in err++ def test_disabled(self, proj_path: Path) -> None:+ """Scrapy leaves the user-configured root handler alone, so the+ message is only logged once, in the user's format."""+ _, _, err = proc(+ "crawl", "sp", "-s", "LOG_INSTALL_ROOT_HANDLER=False", cwd=proj_path+ )+ assert "CUSTOM: It works!" in err+ assert "[sp] INFO: It works!" not in err++class TestMiscCommands(TestProjectBase):def test_list(self, proj_path: Path) -> None:assert call("list", cwd=proj_path) == 0diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 9e22839a1..9bdaa6f38 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -858,7 +858,7 @@ def test_log_scrapy_info(settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture) -> None:with caplog.at_level("INFO"):- CrawlerProcess(settings, install_root_handler=False)+ CrawlerProcess({**settings, "LOG_INSTALL_ROOT_HANDLER": False})assert (caplog.records[0].getMessage()== f"Scrapy {scrapy.__version__} started (bot: scrapybot)"diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.pyindex 9e35a38e2..0b3a52cfe 100644--- a/tests/test_crawler_runners.py+++ b/tests/test_crawler_runners.py@@ -13,6 +13,7 @@ from scrapy.crawler import (CrawlerRunner,CrawlerRunnerBase,)+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_reactor_settings@@ -54,12 +55,13 @@ class TestAsyncCrawlerRunner:class TestCrawlerProcess:def test_crawler_process_accepts_dict(self) -> None:- runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)+ runner = CrawlerProcess({"foo": "bar", "LOG_INSTALL_ROOT_HANDLER": False})assert runner.settings["foo"] == "bar"assert_option_is_default(runner.settings, "RETRY_ENABLED")def test_crawler_process_accepts_None(self) -> None:- runner = CrawlerProcess(install_root_handler=False)+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):+ runner = CrawlerProcess(install_root_handler=False)assert_option_is_default(runner.settings, "RETRY_ENABLED")@@ -67,15 +69,19 @@ class TestCrawlerProcess:class TestAsyncCrawlerProcess:def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:runner = AsyncCrawlerProcess(- {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},- install_root_handler=False,+ {+ "foo": "bar",+ "TWISTED_REACTOR_ENABLED": reactor_pytest != "none",+ "LOG_INSTALL_ROOT_HANDLER": False,+ })assert runner.settings["foo"] == "bar"assert_option_is_default(runner.settings, "RETRY_ENABLED")@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=Falsedef test_crawler_process_accepts_None(self) -> None:- runner = AsyncCrawlerProcess(install_root_handler=False)+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):+ runner = AsyncCrawlerProcess(install_root_handler=False)assert_option_is_default(runner.settings, "RETRY_ENABLED")diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex f4cfd2223..ce8da0b39 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -232,21 +232,27 @@ class TestConfigureLogging:return warnings.showwarning.__module__ == "logging"def test_log_stdout(self) -> None:- configure_logging(settings={"LOG_STDOUT": True}, install_root_handler=False)+ configure_logging(+ settings={"LOG_STDOUT": True, "LOG_INSTALL_ROOT_HANDLER": False}+ )assert isinstance(sys.stdout, StreamLogger)def test_captures_warnings(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(sys, "warnoptions", [])logging.captureWarnings(False)- configure_logging(install_root_handler=False)+ configure_logging(settings={"LOG_INSTALL_ROOT_HANDLER": False})assert self._warnings_are_captured()def test_keeps_warnoptions(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(sys, "warnoptions", ["default"])logging.captureWarnings(False)- configure_logging(install_root_handler=False)+ configure_logging(settings={"LOG_INSTALL_ROOT_HANDLER": False})assert not self._warnings_are_captured()+ def test_install_root_handler_param_deprecated(self) -> None:+ with pytest.warns(ScrapyDeprecationWarning, match="install_root_handler"):+ configure_logging(install_root_handler=False)+def test_reinstall_root_handler_removed_from_root(self) -> None:install_scrapy_root_handler(Settings())handler = get_scrapy_root_handler()diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pyindex 496c65edd..aa6e98e7d 100644--- a/tests/test_zz_resources.py+++ b/tests/test_zz_resources.py@@ -26,8 +26,9 @@ def test_stderr_log_handler() -> None:"""Test that the Scrapy root handler is always properly removed.It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess``- (without ``install_root_handler=False``). It can be removed with- ``_uninstall_scrapy_root_handler()`` if installing it was really needed.+ (unless the ``LOG_INSTALL_ROOT_HANDLER`` setting is ``False``). It can be+ removed with ``_uninstall_scrapy_root_handler()`` if installing it was+ really needed."""c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler)assert c == 0