Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Flexible severity of logging level when items are dropped (#6608)

anubhavca345a3b7390
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 06974f336..8801434d8 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -418,6 +418,38 @@ This setting also affects :setting:`DOWNLOAD_DELAY` and
:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP`
is non-zero, download delay is enforced per IP, not per domain.
+.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
+
+DEFAULT_DROPITEM_LOG_LEVEL
+--------------------------
+
+Default: ``"WARNING"``
+
+Default :ref:`log level <levels>` of messages about dropped items.
+
+When an item is dropped by raising :exc:`scrapy.exceptions.DropItem` from the
+:func:`process_item` method of an :ref:`item pipeline <topics-item-pipeline>`,
+a message is logged, and by default its log level is the one configured in this
+setting.
+
+You may specify this log level as an integer (e.g. ``20``), as a log level
+constant (e.g. ``logging.INFO``) or as a string with the name of a log level
+constant (e.g. ``"INFO"``).
+
+When writing an item pipeline, you can force a different log level by setting
+:attr:`scrapy.exceptions.DropItem.log_level` in your
+:exc:`scrapy.exceptions.DropItem` exception. For example:
+
+.. code-block:: python
+
+ from scrapy.exceptions import DropItem
+
+
+ class MyPipeline:
+ def process_item(self, item, spider):
+ if not item.get("price"):
+ raise DropItem("Missing price data", log_level="INFO")
+ return item
.. setting:: DEFAULT_ITEM_CLASS
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 96566ba86..f37f881a7 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -5,6 +5,8 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add
new exceptions here without documenting them there.
"""
+from __future__ import annotations
+
from typing import Any
# Internal
@@ -58,6 +60,10 @@ class StopDownload(Exception):
class DropItem(Exception):
"""Drop item from the item pipeline"""
+ def __init__(self, message: str, log_level: str | None = None):
+ super().__init__(message)
+ self.log_level = log_level
+
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 76f9c7856..f10e91beb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -120,8 +120,12 @@ class LogFormatter:
spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
+ if (level := getattr(exception, "log_level", None)) is None:
+ level = spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"]
+ if isinstance(level, str):
+ level = getattr(logging, level)
return {
- "level": logging.WARNING,
+ "level": level,
"msg": DROPPEDMSG,
"args": {
"exception": exception,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0bbde118e..7ef365f68 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -49,6 +49,8 @@ CONCURRENT_REQUESTS_PER_IP = 0
COOKIES_ENABLED = True
COOKIES_DEBUG = False
+DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"
+
DEFAULT_ITEM_CLASS = "scrapy.item.Item"
DEFAULT_REQUEST_HEADERS = {
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 61a9f3f8d..e5d077858 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,5 +1,7 @@
+import logging
import unittest
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.python.failure import Failure
@@ -26,6 +28,7 @@ class LogFormatterTestCase(unittest.TestCase):
def setUp(self):
self.formatter = LogFormatter()
self.spider = Spider("default")
+ self.spider.crawler = get_crawler()
def test_crawled_with_referer(self):
req = Request("http://www.example.com")
@@ -68,6 +71,62 @@ class LogFormatterTestCase(unittest.TestCase):
assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, ["Dropped: \u2018", "{}"])
+ def test_dropitem_default_log_level(self):
+ item = {}
+ exception = DropItem("Test drop")
+ response = Response("http://www.example.com")
+ spider = Spider("foo")
+ spider.crawler = get_crawler(Spider)
+
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], logging.WARNING)
+
+ spider.crawler.settings.frozen = False
+ spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO
+ spider.crawler.settings.frozen = True
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], logging.INFO)
+
+ spider.crawler.settings.frozen = False
+ spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO"
+ spider.crawler.settings.frozen = True
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], logging.INFO)
+
+ spider.crawler.settings.frozen = False
+ spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10
+ spider.crawler.settings.frozen = True
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], logging.DEBUG)
+
+ spider.crawler.settings.frozen = False
+ spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0
+ spider.crawler.settings.frozen = True
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], logging.NOTSET)
+
+ unsupported_value = object()
+ spider.crawler.settings.frozen = False
+ spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value
+ spider.crawler.settings.frozen = True
+ logkws = self.formatter.dropped(item, exception, response, spider)
+ self.assertEqual(logkws["level"], unsupported_value)
+
+ with pytest.raises(TypeError):
+ logging.log(logkws["level"], "message")
+
+ def test_dropitem_custom_log_level(self):
+ item = {}
+ response = Response("http://www.example.com")
+
+ exception = DropItem("Test drop", log_level="INFO")
+ logkws = self.formatter.dropped(item, exception, response, self.spider)
+ self.assertEqual(logkws["level"], logging.INFO)
+
+ exception = DropItem("Test drop", log_level="ERROR")
+ logkws = self.formatter.dropped(item, exception, response, self.spider)
+ self.assertEqual(logkws["level"], logging.ERROR)
+
def test_item_error(self):
# In practice, the complete traceback is shown by passing the
# 'exc_info' argument to the logging function
@@ -145,6 +204,7 @@ class LogformatterSubclassTest(LogFormatterTestCase):
def setUp(self):
self.formatter = LogFormatterSubclass()
self.spider = Spider("default")
+ self.spider.crawler = get_crawler(Spider)
def test_crawled_with_referer(self):
req = Request("http://www.example.com")