Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add LogFormatter.error method

Eugenio Lacuestaf6872189b965
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 1f389cf2e..3273a1506 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -231,9 +231,9 @@ class Scraper(object):
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
- logger.error('Error processing %(item)s', {'item': item},
- exc_info=failure_to_exc_info(output),
- extra={'spider': spider})
+ logkws = self.logformatter.error(item, ex, response, spider)
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
+ exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index f15940ed1..4437d1106 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
+ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
@@ -92,6 +93,16 @@ class LogFormatter(object):
}
}
+ def error(self, item, exception, response, spider):
+ """Logs a message when an item causes an error while it is passing through the item pipeline."""
+ return {
+ 'level': logging.ERROR,
+ 'msg': ERRORMSG,
+ 'args': {
+ 'item': item,
+ }
+ }
+
@classmethod
def from_crawler(cls, crawler):
return cls()