Scrapy, a fast high-level web crawling & scraping framework for Python.
Support %(spider)s in LOG_FORMAT (#8086)
Adriancd8425a50e72
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rstindex dc6a5c127..b05030a2e 100644--- a/docs/topics/logging.rst+++ b/docs/topics/logging.rst@@ -118,8 +118,18 @@ instance, which can be accessed and used like this:def parse(self, response):self.logger.info("Parse function called on %s", response.url)-That logger is created using the Spider's name, but you can use any custom-Python logger you want. For example:+That logger is created using the Spider's name, and every message logged+through it, as well as most messages logged by Scrapy itself while crawling,+carries a reference to the spider that triggered it. This is useful to tell+spiders apart when :ref:`running several of them in the same process+<run-multiple-spiders>`: add ``%(spider)s`` to :setting:`LOG_FORMAT` to+include the spider name in every log message, e.g.:++.. code-block:: python++ LOG_FORMAT = "%(asctime)s [%(name)s] %(spider)s %(levelname)s: %(message)s"++You can use any custom Python logger you want too. For example:.. code-block:: pythondiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 639f7ef8f..2aba03db5 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1745,7 +1745,9 @@ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``String for formatting log messages. Refer to the:ref:`Python logging documentation <logrecord-attributes>` for the whole-list of available placeholders.+list of available placeholders, plus ``%(spider)s`` for the name of the+spider that triggered the log message, or ``"-"`` for messages not tied to+a spider (see :ref:`topics-logging-from-spiders`)... note:: This is a :ref:`logging setting <logging-settings>`.diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex b9399eb0f..d3c575468 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -445,4 +445,4 @@ def _find_method(obj: Any, func: Callable[..., Any]) -> str:# https://docs.python.org/3/reference/datamodel.htmlif obj_func.__func__ is func.__func__:return name- raise ValueError(f"Function {func} is not an instance method in: {obj}")+ raise ValueError(f"Function {func} is not an instance method in: {obj!r}")diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 115b9d849..c7ffb5980 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -76,7 +76,7 @@ class MiddlewareManager(ABC):elif self._compat_spider is not spider:raise RuntimeError(f"Different instances of Spider were passed to {type(self).__name__}:"- f" {self._compat_spider} and {spider}"+ f" {self._compat_spider!r} and {spider!r}")@classmethoddiff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 6244e3264..d16cc8e40 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -181,6 +181,9 @@ class Spider(object_ref):def __repr__(self) -> str:return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"+ def __str__(self) -> str:+ return self.name+# Top-level importsfrom scrapy.spiders.crawl import CrawlSpider, Rulediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 1d56ff5e1..f94abfa07 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -66,6 +66,19 @@ class TopLevelFormatter(logging.Filter):return True+class _SpiderRecordDefaultFilter(logging.Filter):+ """Default the ``spider`` log record attribute to ``"-"``.++ This lets :setting:`LOG_FORMAT` reference ``%(spider)s`` even for log+ records that were not given a ``spider`` via ``extra``.+ """++ def filter(self, record: logging.LogRecord) -> bool:+ if not hasattr(record, "spider"):+ record.spider = "-"+ return True++DEFAULT_LOGGING = {"version": 1,"disable_existing_loggers": False,@@ -213,6 +226,7 @@ def _get_handler(settings: Settings) -> logging.Handler:handler.setFormatter(_get_formatter(handler, settings))handler.setLevel(settings.get("LOG_LEVEL"))+ handler.addFilter(_SpiderRecordDefaultFilter())if settings.getbool("LOG_SHORT_NAMES"):handler.addFilter(TopLevelFormatter(["scrapy"]))return handlerdiff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex f10c76ac4..24db88b49 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -184,7 +184,7 @@ def _get_method(obj: Any, name: object) -> Any:try:return getattr(obj, name)except AttributeError:- raise ValueError(f"Method {name!r} not found in: {obj}") from None+ raise ValueError(f"Method {name!r} not found in: {obj!r}") from Nonedef _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]:diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 825d281d9..55acfe890 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport jsonimport logging-import reimport sysimport warningsfrom io import StringIO@@ -20,6 +19,7 @@ from scrapy.utils.log import (StreamLogger,TopLevelFormatter,_get_formatter,+ _get_handler,_uninstall_scrapy_root_handler,configure_logging,failure_to_exc_info,@@ -77,6 +77,32 @@ class TestTopLevelFormatter:assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples+class TestSpiderPlaceholderInLogFormat:+ def _log(self, log_format: str, extra: dict[str, Any] | None) -> str:+ stream = StringIO()+ settings = Settings({"LOG_FORMAT": log_format})+ handler = _get_handler(settings)+ assert isinstance(handler, logging.StreamHandler)+ handler.stream = stream+ logger = logging.getLogger("test_spider_placeholder")+ logger.addHandler(handler)+ logger.setLevel(logging.DEBUG)+ try:+ logger.info("test log msg", extra=extra)+ finally:+ logger.removeHandler(handler)+ return stream.getvalue()++ def test_default_placeholder(self) -> None:+ assert self._log("%(spider)s %(message)s", None) == "- test log msg\n"++ def test_spider_name(self) -> None:+ spider = LogSpider()+ assert self._log("%(spider)s %(message)s", {"spider": spider}) == (+ "log_spider test log msg\n"+ )++class TestLogCounterHandler:@pytest.fixturedef crawler(self) -> Crawler:@@ -357,8 +383,6 @@ class TestLogging:class TestLoggingWithExtra:- regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")-@pytest.fixturedef log_stream(self) -> StringIO:return StringIO()@@ -391,7 +415,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "DEBUG"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:@@ -403,7 +427,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "INFO"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:@@ -415,7 +439,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "WARNING"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:@@ -427,7 +451,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "ERROR"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:@@ -439,7 +463,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "CRITICAL"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]def test_overwrite_spider_extra(@@ -453,7 +477,7 @@ class TestLoggingWithExtra:assert log_contents["levelname"] == "ERROR"assert log_contents["message"] == log_message- assert self.regex_pattern.match(log_contents["spider"])+ assert log_contents["spider"] == "log_spider"assert log_contents["important_info"] == extra["important_info"]