Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

LogFormatter: Add the ability to skip log messages (#3987)

watsta0b52fa6ca9ec
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 37fe0a873..fa913e528 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -233,10 +233,11 @@ class ExecutionEngine(object):
def _on_success(response):
assert isinstance(response, (Response, Request))
if isinstance(response, Response):
- response.request = request # tie request to response received
+ response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
- self.signals.send_catch_log(signal=signals.response_received, \
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ self.signals.send_catch_log(signal=signals.response_received,
response=response, request=request, spider=spider)
return response
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 66f5d0e05..1f389cf2e 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -225,7 +225,8 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
@@ -238,7 +239,8 @@ class Scraper(object):
spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index b4d6787ff..f15940ed1 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -29,6 +29,10 @@ class LogFormatter(object):
* ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
The final log message is computed as ``msg % args``.
+ Users can define their own ``LogFormatter`` class if they want to customise how
+ each action is logged or if they want to omit it entirely. In order to omit
+ logging an action the method must return ``None``.
+
Here is an example on how to create a custom log formatter to lower the severity level of
the log message when an item is dropped from the pipeline::
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 94e6c9fde..eb9c4a561 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,10 +1,18 @@
import unittest
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TwistedTestCase
import six
-from scrapy.spiders import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import DropItem
from scrapy.http import Request, Response
from scrapy.item import Item, Field
from scrapy.logformatter import LogFormatter
+from scrapy.spiders import Spider
+from tests.mockserver import MockServer
+from tests.spiders import ItemSpider
class CustomItem(Item):
@@ -89,5 +97,61 @@ class LogformatterSubclassTest(LoggingContribTest):
pass
+class SkipMessagesLogFormatter(LogFormatter):
+ def crawled(self, *args, **kwargs):
+ return None
+
+ def scraped(self, *args, **kwargs):
+ return None
+
+ def dropped(self, *args, **kwargs):
+ return None
+
+
+class DropSomeItemsPipeline(object):
+ drop = True
+
+ def process_item(self, item, spider):
+ if self.drop:
+ self.drop = False
+ raise DropItem("Ignoring item")
+ else:
+ self.drop = True
+
+class ShowOrSkipMessagesTestCase(TwistedTestCase):
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+ self.base_settings = {
+ 'LOG_LEVEL': 'DEBUG',
+ 'ITEM_PIPELINES': {
+ __name__ + '.DropSomeItemsPipeline': 300,
+ },
+ }
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ @defer.inlineCallbacks
+ def test_show_messages(self):
+ crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+ with LogCapture() as lc:
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
+ self.assertIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+ self.assertIn("Dropped: Ignoring item", str(lc))
+
+ @defer.inlineCallbacks
+ def test_skip_messages(self):
+ settings = self.base_settings.copy()
+ settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+ crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+ with LogCapture() as lc:
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
+ self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+ self.assertNotIn("Dropped: Ignoring item", str(lc))
+
+
if __name__ == "__main__":
unittest.main()