Scrapy, a fast high-level web crawling & scraping framework for Python.
Raise log level for HttpErrorMiddleware to INFO (from DEBUG)
Paul Tremberth81a0e3cd9395
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 6b7c50fce..e34c265c1 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -46,7 +46,7 @@ class HttpErrorMiddleware(object):def process_spider_exception(self, response, exception, spider):if isinstance(exception, HttpError):- logger.debug(+ logger.info("Ignoring response %(response)r: HTTP status code is not handled or not allowed",{'response': response}, extra={'spider': spider},)diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex a64400482..319746350 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -1,3 +1,4 @@+import loggingfrom unittest import TestCasefrom testfixtures import LogCapture@@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):self.assertIn('Ignoring response <500', str(log))self.assertNotIn('Ignoring response <200', str(log))self.assertNotIn('Ignoring response <402', str(log))++ @defer.inlineCallbacks+ def test_logging_level(self):+ # HttpError logs ignored responses with level INFO+ crawler = get_crawler(_HttpErrorSpider)+ with LogCapture(level=logging.INFO) as log:+ yield crawler.crawl()+ self.assertEqual(crawler.spider.parsed, {'200'})+ self.assertEqual(crawler.spider.failed, {'404', '402', '500'})++ self.assertIn('Ignoring response <402', str(log))+ self.assertIn('Ignoring response <404', str(log))+ self.assertIn('Ignoring response <500', str(log))+ self.assertNotIn('Ignoring response <200', str(log))++ # with level WARNING, we shouldn't capture anything from HttpError+ crawler = get_crawler(_HttpErrorSpider)+ with LogCapture(level=logging.WARNING) as log:+ yield crawler.crawl()+ self.assertEqual(crawler.spider.parsed, {'200'})+ self.assertEqual(crawler.spider.failed, {'404', '402', '500'})++ self.assertNotIn('Ignoring response <402', str(log))+ self.assertNotIn('Ignoring response <404', str(log))+ self.assertNotIn('Ignoring response <500', str(log))+ self.assertNotIn('Ignoring response <200', str(log))