Scrapy, a fast high-level web crawling & scraping framework for Python.
LogFormatter: Add the ability to skip log messages (#3987)
watsta0b52fa6ca9ec
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 37fe0a873..fa913e528 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -233,10 +233,11 @@ class ExecutionEngine(object):def _on_success(response):assert isinstance(response, (Response, Request))if isinstance(response, Response):- response.request = request # tie request to response received+ response.request = request # tie request to response receivedlogkws = self.logformatter.crawled(request, response, spider)- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})- self.signals.send_catch_log(signal=signals.response_received, \+ if logkws is not None:+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})+ self.signals.send_catch_log(signal=signals.response_received,response=response, request=request, spider=spider)return responsediff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 66f5d0e05..1f389cf2e 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -225,7 +225,8 @@ class Scraper(object):ex = output.valueif isinstance(ex, DropItem):logkws = self.logformatter.dropped(item, ex, response, spider)- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})+ if logkws is not None:+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})return self.signals.send_catch_log_deferred(signal=signals.item_dropped, item=item, response=response,spider=spider, exception=output.value)@@ -238,7 +239,8 @@ class Scraper(object):spider=spider, failure=output)else:logkws = self.logformatter.scraped(output, response, spider)- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})+ if logkws is not None:+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})return self.signals.send_catch_log_deferred(signal=signals.item_scraped, item=output, response=response,spider=spider)diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex b4d6787ff..f15940ed1 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -29,6 +29,10 @@ class LogFormatter(object):* ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.The final log message is computed as ``msg % args``.+ Users can define their own ``LogFormatter`` class if they want to customise how+ each action is logged or if they want to omit it entirely. In order to omit+ logging an action the method must return ``None``.+Here is an example on how to create a custom log formatter to lower the severity level ofthe log message when an item is dropped from the pipeline::diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 94e6c9fde..eb9c4a561 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -1,10 +1,18 @@import unittest++from testfixtures import LogCapture+from twisted.internet import defer+from twisted.trial.unittest import TestCase as TwistedTestCaseimport six-from scrapy.spiders import Spider+from scrapy.crawler import CrawlerRunner+from scrapy.exceptions import DropItemfrom scrapy.http import Request, Responsefrom scrapy.item import Item, Fieldfrom scrapy.logformatter import LogFormatter+from scrapy.spiders import Spider+from tests.mockserver import MockServer+from tests.spiders import ItemSpiderclass CustomItem(Item):@@ -89,5 +97,61 @@ class LogformatterSubclassTest(LoggingContribTest):pass+class SkipMessagesLogFormatter(LogFormatter):+ def crawled(self, *args, **kwargs):+ return None++ def scraped(self, *args, **kwargs):+ return None++ def dropped(self, *args, **kwargs):+ return None+++class DropSomeItemsPipeline(object):+ drop = True++ def process_item(self, item, spider):+ if self.drop:+ self.drop = False+ raise DropItem("Ignoring item")+ else:+ self.drop = True++class ShowOrSkipMessagesTestCase(TwistedTestCase):+ def setUp(self):+ self.mockserver = MockServer()+ self.mockserver.__enter__()+ self.base_settings = {+ 'LOG_LEVEL': 'DEBUG',+ 'ITEM_PIPELINES': {+ __name__ + '.DropSomeItemsPipeline': 300,+ },+ }++ def tearDown(self):+ self.mockserver.__exit__(None, None, None)++ @defer.inlineCallbacks+ def test_show_messages(self):+ crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)+ with LogCapture() as lc:+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))+ self.assertIn("Crawled (200) <GET http://127.0.0.1:", str(lc))+ self.assertIn("Dropped: Ignoring item", str(lc))++ @defer.inlineCallbacks+ def test_skip_messages(self):+ settings = self.base_settings.copy()+ settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'+ crawler = CrawlerRunner(settings).create_crawler(ItemSpider)+ with LogCapture() as lc:+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))+ self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc))+ self.assertNotIn("Dropped: Ignoring item", str(lc))++if __name__ == "__main__":unittest.main()