Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Raise log level for HttpErrorMiddleware to INFO (from DEBUG)

Paul Tremberth81a0e3cd9395
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 6b7c50fce..e34c265c1 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -46,7 +46,7 @@ class HttpErrorMiddleware(object):
def process_spider_exception(self, response, exception, spider):
if isinstance(exception, HttpError):
- logger.debug(
+ logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{'response': response}, extra={'spider': spider},
)
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index a64400482..319746350 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,3 +1,4 @@
+import logging
from unittest import TestCase
from testfixtures import LogCapture
@@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
self.assertIn('Ignoring response <500', str(log))
self.assertNotIn('Ignoring response <200', str(log))
self.assertNotIn('Ignoring response <402', str(log))
+
+ @defer.inlineCallbacks
+ def test_logging_level(self):
+ # HttpError logs ignored responses with level INFO
+ crawler = get_crawler(_HttpErrorSpider)
+ with LogCapture(level=logging.INFO) as log:
+ yield crawler.crawl()
+ self.assertEqual(crawler.spider.parsed, {'200'})
+ self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+
+ self.assertIn('Ignoring response <402', str(log))
+ self.assertIn('Ignoring response <404', str(log))
+ self.assertIn('Ignoring response <500', str(log))
+ self.assertNotIn('Ignoring response <200', str(log))
+
+ # with level WARNING, we shouldn't capture anything from HttpError
+ crawler = get_crawler(_HttpErrorSpider)
+ with LogCapture(level=logging.WARNING) as log:
+ yield crawler.crawl()
+ self.assertEqual(crawler.spider.parsed, {'200'})
+ self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+
+ self.assertNotIn('Ignoring response <402', str(log))
+ self.assertNotIn('Ignoring response <404', str(log))
+ self.assertNotIn('Ignoring response <500', str(log))
+ self.assertNotIn('Ignoring response <200', str(log))