Scrapy, a fast high-level web crawling & scraping framework for Python.
Add LOG_FILE_APPEND to settings
Erik Kempermanca320feb2afa
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rstindex dda04dc4d..d593c74c6 100644--- a/docs/topics/logging.rst+++ b/docs/topics/logging.rst@@ -143,6 +143,7 @@ Logging settingsThese settings can be used to configure the logging:* :setting:`LOG_FILE`+* :setting:`LOG_FILE_APPEND`* :setting:`LOG_ENABLED`* :setting:`LOG_ENCODING`* :setting:`LOG_LEVEL`@@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If:setting:`LOG_FILE` is set, messages sent through the root logger will beredirected to a file named :setting:`LOG_FILE` with encoding:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log-messages will be displayed on the standard error. Lastly, if+messages will be displayed on the standard error. If :setting:`LOG_FILE` is set+and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten+(discarding the output from previous runs, if any). Lastly, if:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.:setting:`LOG_LEVEL` determines the minimum level of severity to display, thosediff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex e63aca312..210c1def7 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1026,6 +1026,16 @@ Default: ``None``File name to use for logging output. If ``None``, standard error will be used.+.. setting:: LOG_FILE_APPEND++LOG_FILE_APPEND+---------------++Default: ``True``++If ``False``, the log file specified with :setting:`LOG_FILE` will be+overwritten (discarding the output from previous runs, if any).+.. setting:: LOG_FORMATLOG_FORMATdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 4ef330dd2..8389a70cb 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'LOG_STDOUT = FalseLOG_LEVEL = 'DEBUG'LOG_FILE = None+LOG_FILE_APPEND = TrueLOG_SHORT_NAMES = FalseSCHEDULER_DEBUG = Falsediff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 6c456ed60..0441c0358 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -124,8 +124,9 @@ def _get_handler(settings):""" Return a log handler object according to settings """filename = settings.get('LOG_FILE')if filename:+ mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'encoding = settings.get('LOG_ENCODING')- handler = logging.FileHandler(filename, encoding=encoding)+ handler = logging.FileHandler(filename, mode=mode, encoding=encoding)elif settings.getbool('LOG_ENABLED'):handler = logging.StreamHandler()else:diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex dec517bb6..a80ad4388 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):def test_spider_custom_settings_log_level(self):log_file = self.mktemp()+ with open(log_file, 'wb') as fo:+ fo.write('previous message\n'.encode('utf-8'))class MySpider(scrapy.Spider):name = 'spider'@@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase):logging.error('error message')with open(log_file, 'rb') as fo:- logged = fo.read().decode('utf8')+ logged = fo.read().decode('utf-8')+ self.assertIn('previous message', logged)self.assertNotIn('debug message', logged)self.assertIn('info message', logged)self.assertIn('warning message', logged)@@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase):crawler.stats.get_value('log_count/INFO') - info_count, 1)self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)+ def test_spider_custom_settings_log_append(self):+ log_file = self.mktemp()+ with open(log_file, 'wb') as fo:+ fo.write('previous message\n'.encode('utf-8'))++ class MySpider(scrapy.Spider):+ name = 'spider'+ custom_settings = {+ 'LOG_FILE': log_file,+ 'LOG_FILE_APPEND': False,+ # disable telnet if not available to avoid an extra warning+ 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,+ }++ configure_logging()+ crawler = Crawler(MySpider, {})+ logging.debug('debug message')++ with open(log_file, 'rb') as fo:+ logged = fo.read().decode('utf-8')++ self.assertNotIn('previous message', logged)+ self.assertIn('debug message', logged)+class SpiderLoaderWithWrongInterface: