Scrapy, a fast high-level web crawling & scraping framework for Python.
Respect log settings in custom_settings: fixes GH-1612
Konstantin Lopuhinef04cfd237ec
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 443a9aa2f..7b8518832 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -16,7 +16,9 @@ from scrapy.signalmanager import SignalManagerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_namesfrom scrapy.utils.misc import load_object-from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info+from scrapy.utils.log import (+ LogCounterHandler, configure_logging, log_scrapy_info,+ get_scrapy_root_handler, install_scrapy_root_handler)from scrapy import signalslogger = logging.getLogger(__name__)@@ -35,8 +37,11 @@ class Crawler(object):self.signals = SignalManager(self)self.stats = load_object(self.settings['STATS_CLASS'])(self)- handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))+ handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))logging.root.addHandler(handler)+ if get_scrapy_root_handler() is not None:+ # scrapy root handler alread installed: update it with new settings+ install_scrapy_root_handler(self.settings)# lambda is assigned to Crawler attribute because this way it is not# garbage collected after leaving __init__ scopeself.__remove_handler = lambda: logging.root.removeHandler(handler)diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex f33ce7017..6ceb61a82 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -96,9 +96,25 @@ def configure_logging(settings=None, install_root_handler=True):sys.stdout = StreamLogger(logging.getLogger('stdout'))if install_root_handler:- logging.root.setLevel(logging.NOTSET)- handler = _get_handler(settings)- logging.root.addHandler(handler)+ install_scrapy_root_handler(settings)+++def install_scrapy_root_handler(settings):+ global _scrapy_root_handler++ if (_scrapy_root_handler is not None+ and _scrapy_root_handler in logging.root.handlers):+ logging.root.removeHandler(_scrapy_root_handler)+ logging.root.setLevel(logging.NOTSET)+ _scrapy_root_handler = _get_handler(settings)+ logging.root.addHandler(_scrapy_root_handler)+++def get_scrapy_root_handler():+ return _scrapy_root_handler+++_scrapy_root_handler = Nonedef _get_handler(settings):diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex d5babdded..3c5d9b958 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -97,8 +97,8 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_bug_before_yield(self):+ crawler = self.runner.create_crawler(BrokenStartRequestsSpider)with LogCapture('scrapy', level=logging.ERROR) as l:- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_before_yield=1)self.assertEqual(len(l.records), 1)@@ -108,8 +108,8 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_bug_yielding(self):+ crawler = self.runner.create_crawler(BrokenStartRequestsSpider)with LogCapture('scrapy', level=logging.ERROR) as l:- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_yielding=1)self.assertEqual(len(l.records), 1)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 53a1202e3..ba0d709ff 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,3 +1,6 @@+import logging+import os+import tempfileimport warningsimport unittest@@ -5,6 +8,7 @@ import scrapyfrom scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcessfrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoader+from scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.misc import load_objectfrom scrapy.extensions.throttle import AutoThrottle@@ -74,6 +78,48 @@ class SpiderSettingsTestCase(unittest.TestCase):self.assertIn(AutoThrottle, enabled_exts)+class CrawlerLoggingTestCase(unittest.TestCase):+ def test_no_root_handler_installed(self):+ handler = get_scrapy_root_handler()+ if handler is not None:+ logging.root.removeHandler(handler)++ class MySpider(scrapy.Spider):+ name = 'spider'++ crawler = Crawler(MySpider, {})+ assert get_scrapy_root_handler() is None++ def test_spider_custom_settings_log_level(self):+ with tempfile.NamedTemporaryFile() as log_file:+ class MySpider(scrapy.Spider):+ name = 'spider'+ custom_settings = {+ 'LOG_LEVEL': 'INFO',+ 'LOG_FILE': log_file.name,+ }++ configure_logging()+ self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)+ crawler = Crawler(MySpider, {})+ self.assertEqual(get_scrapy_root_handler().level, logging.INFO)+ info_count = crawler.stats.get_value('log_count/INFO')+ logging.debug('debug message')+ logging.info('info message')+ logging.warning('warning message')+ logging.error('error message')+ logged = log_file.read().decode('utf8')+ self.assertNotIn('debug message', logged)+ self.assertIn('info message', logged)+ self.assertIn('warning message', logged)+ self.assertIn('error message', logged)+ self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1)+ self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1)+ self.assertEqual(+ crawler.stats.get_value('log_count/INFO') - info_count, 1)+ self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)++class SpiderLoaderWithWrongInterface(object):def unneeded_method(self):