Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add LOG_FILE_APPEND to settings

Erik Kempermanca320feb2afa
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index dda04dc4d..d593c74c6 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -143,6 +143,7 @@ Logging settings
These settings can be used to configure the logging:
* :setting:`LOG_FILE`
+* :setting:`LOG_FILE_APPEND`
* :setting:`LOG_ENABLED`
* :setting:`LOG_ENCODING`
* :setting:`LOG_LEVEL`
@@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
redirected to a file named :setting:`LOG_FILE` with encoding
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
-messages will be displayed on the standard error. Lastly, if
+messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
+and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
+(discarding the output from previous runs, if any). Lastly, if
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e63aca312..210c1def7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1026,6 +1026,16 @@ Default: ``None``
File name to use for logging output. If ``None``, standard error will be used.
+.. setting:: LOG_FILE_APPEND
+
+LOG_FILE_APPEND
+---------------
+
+Default: ``True``
+
+If ``False``, the log file specified with :setting:`LOG_FILE` will be
+overwritten (discarding the output from previous runs, if any).
+
.. setting:: LOG_FORMAT
LOG_FORMAT
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 4ef330dd2..8389a70cb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
LOG_STDOUT = False
LOG_LEVEL = 'DEBUG'
LOG_FILE = None
+LOG_FILE_APPEND = True
LOG_SHORT_NAMES = False
SCHEDULER_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6c456ed60..0441c0358 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -124,8 +124,9 @@ def _get_handler(settings):
""" Return a log handler object according to settings """
filename = settings.get('LOG_FILE')
if filename:
+ mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
encoding = settings.get('LOG_ENCODING')
- handler = logging.FileHandler(filename, encoding=encoding)
+ handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
elif settings.getbool('LOG_ENABLED'):
handler = logging.StreamHandler()
else:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index dec517bb6..a80ad4388 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
def test_spider_custom_settings_log_level(self):
log_file = self.mktemp()
+ with open(log_file, 'wb') as fo:
+ fo.write('previous message\n'.encode('utf-8'))
class MySpider(scrapy.Spider):
name = 'spider'
@@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase):
logging.error('error message')
with open(log_file, 'rb') as fo:
- logged = fo.read().decode('utf8')
+ logged = fo.read().decode('utf-8')
+ self.assertIn('previous message', logged)
self.assertNotIn('debug message', logged)
self.assertIn('info message', logged)
self.assertIn('warning message', logged)
@@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase):
crawler.stats.get_value('log_count/INFO') - info_count, 1)
self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
+ def test_spider_custom_settings_log_append(self):
+ log_file = self.mktemp()
+ with open(log_file, 'wb') as fo:
+ fo.write('previous message\n'.encode('utf-8'))
+
+ class MySpider(scrapy.Spider):
+ name = 'spider'
+ custom_settings = {
+ 'LOG_FILE': log_file,
+ 'LOG_FILE_APPEND': False,
+ # disable telnet if not available to avoid an extra warning
+ 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+ }
+
+ configure_logging()
+ crawler = Crawler(MySpider, {})
+ logging.debug('debug message')
+
+ with open(log_file, 'rb') as fo:
+ logged = fo.read().decode('utf-8')
+
+ self.assertNotIn('previous message', logged)
+ self.assertIn('debug message', logged)
+
class SpiderLoaderWithWrongInterface: