Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support %(spider)s in LOG_FORMAT (#8086)

Adriancd8425a50e72
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index dc6a5c127..b05030a2e 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -118,8 +118,18 @@ instance, which can be accessed and used like this:
def parse(self, response):
self.logger.info("Parse function called on %s", response.url)
-That logger is created using the Spider's name, but you can use any custom
-Python logger you want. For example:
+That logger is created using the Spider's name, and every message logged
+through it, as well as most messages logged by Scrapy itself while crawling,
+carries a reference to the spider that triggered it. This is useful to tell
+spiders apart when :ref:`running several of them in the same process
+<run-multiple-spiders>`: add ``%(spider)s`` to :setting:`LOG_FORMAT` to
+include the spider name in every log message, e.g.:
+
+.. code-block:: python
+
+ LOG_FORMAT = "%(asctime)s [%(name)s] %(spider)s %(levelname)s: %(message)s"
+
+You can use any custom Python logger you want too. For example:
.. code-block:: python
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 639f7ef8f..2aba03db5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1745,7 +1745,9 @@ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the whole
-list of available placeholders.
+list of available placeholders, plus ``%(spider)s`` for the name of the
+spider that triggered the log message, or ``"-"`` for messages not tied to
+a spider (see :ref:`topics-logging-from-spiders`).
.. note:: This is a :ref:`logging setting <logging-settings>`.
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index b9399eb0f..d3c575468 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -445,4 +445,4 @@ def _find_method(obj: Any, func: Callable[..., Any]) -> str:
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
- raise ValueError(f"Function {func} is not an instance method in: {obj}")
+ raise ValueError(f"Function {func} is not an instance method in: {obj!r}")
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 115b9d849..c7ffb5980 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -76,7 +76,7 @@ class MiddlewareManager(ABC):
elif self._compat_spider is not spider:
raise RuntimeError(
f"Different instances of Spider were passed to {type(self).__name__}:"
- f" {self._compat_spider} and {spider}"
+ f" {self._compat_spider!r} and {spider!r}"
)
@classmethod
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 6244e3264..d16cc8e40 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -181,6 +181,9 @@ class Spider(object_ref):
def __repr__(self) -> str:
return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"
+ def __str__(self) -> str:
+ return self.name
+
# Top-level imports
from scrapy.spiders.crawl import CrawlSpider, Rule
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 1d56ff5e1..f94abfa07 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -66,6 +66,19 @@ class TopLevelFormatter(logging.Filter):
return True
+class _SpiderRecordDefaultFilter(logging.Filter):
+ """Default the ``spider`` log record attribute to ``"-"``.
+
+ This lets :setting:`LOG_FORMAT` reference ``%(spider)s`` even for log
+ records that were not given a ``spider`` via ``extra``.
+ """
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ if not hasattr(record, "spider"):
+ record.spider = "-"
+ return True
+
+
DEFAULT_LOGGING = {
"version": 1,
"disable_existing_loggers": False,
@@ -213,6 +226,7 @@ def _get_handler(settings: Settings) -> logging.Handler:
handler.setFormatter(_get_formatter(handler, settings))
handler.setLevel(settings.get("LOG_LEVEL"))
+ handler.addFilter(_SpiderRecordDefaultFilter())
if settings.getbool("LOG_SHORT_NAMES"):
handler.addFilter(TopLevelFormatter(["scrapy"]))
return handler
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index f10c76ac4..24db88b49 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -184,7 +184,7 @@ def _get_method(obj: Any, name: object) -> Any:
try:
return getattr(obj, name)
except AttributeError:
- raise ValueError(f"Method {name!r} not found in: {obj}") from None
+ raise ValueError(f"Method {name!r} not found in: {obj!r}") from None
def _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]:
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 825d281d9..55acfe890 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import json
import logging
-import re
import sys
import warnings
from io import StringIO
@@ -20,6 +19,7 @@ from scrapy.utils.log import (
StreamLogger,
TopLevelFormatter,
_get_formatter,
+ _get_handler,
_uninstall_scrapy_root_handler,
configure_logging,
failure_to_exc_info,
@@ -77,6 +77,32 @@ class TestTopLevelFormatter:
assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples
+class TestSpiderPlaceholderInLogFormat:
+ def _log(self, log_format: str, extra: dict[str, Any] | None) -> str:
+ stream = StringIO()
+ settings = Settings({"LOG_FORMAT": log_format})
+ handler = _get_handler(settings)
+ assert isinstance(handler, logging.StreamHandler)
+ handler.stream = stream
+ logger = logging.getLogger("test_spider_placeholder")
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+ try:
+ logger.info("test log msg", extra=extra)
+ finally:
+ logger.removeHandler(handler)
+ return stream.getvalue()
+
+ def test_default_placeholder(self) -> None:
+ assert self._log("%(spider)s %(message)s", None) == "- test log msg\n"
+
+ def test_spider_name(self) -> None:
+ spider = LogSpider()
+ assert self._log("%(spider)s %(message)s", {"spider": spider}) == (
+ "log_spider test log msg\n"
+ )
+
+
class TestLogCounterHandler:
@pytest.fixture
def crawler(self) -> Crawler:
@@ -357,8 +383,6 @@ class TestLogging:
class TestLoggingWithExtra:
- regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
-
@pytest.fixture
def log_stream(self) -> StringIO:
return StringIO()
@@ -391,7 +415,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "DEBUG"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]
def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
@@ -403,7 +427,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "INFO"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]
def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
@@ -415,7 +439,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "WARNING"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]
def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
@@ -427,7 +451,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]
def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
@@ -439,7 +463,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "CRITICAL"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]
def test_overwrite_spider_extra(
@@ -453,7 +477,7 @@ class TestLoggingWithExtra:
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
- assert self.regex_pattern.match(log_contents["spider"])
+ assert log_contents["spider"] == "log_spider"
assert log_contents["important_info"] == extra["important_info"]