Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

log: add log rotation to --log-file - fixes #2259

Nick Craig-Wood8d353039a694
diff --git a/docs/content/docs.md b/docs/content/docs.md
index de5b9590d..ba9435781 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -1727,6 +1727,57 @@ Note that if you are using the `logrotate` program to manage rclone's
logs, then you should use the `copytruncate` option as rclone doesn't
have a signal to rotate logs.
+Alternatively you can use the options below to manage rclone's built
+in log rotation.
+
+### --log-file-max-size SizeSuffix
+
+Maximum size of the log file before it's rotated (eg `10M`). This SizeSuffix
+is rounded to the nearest MiB or 1 MiB if lower.
+
+If `--log-file` is not set then this option will be ignored.
+
+If this option is not set, then the other log rotation options will be
+ignored.
+
+For example if the following flags are in use
+
+```sh
+rclone --log-file rclone.log --log-file-max-size 1M --log-file-max-backups 3
+```
+
+Then this will create log files which look like this
+
+```console
+$ ls -l
+-rw------- 1 user user 1048491 Apr 11 17:15 rclone-2025-04-11T17-15-29.998.log
+-rw------- 1 user user 1048511 Apr 11 17:15 rclone-2025-04-11T17-15-30.467.log
+-rw------- 1 user user 1048559 Apr 11 17:15 rclone-2025-04-11T17-15-30.543.log
+-rw------- 1 user user 521602 Apr 11 17:15 rclone.log
+```
+
+The file `rclone.log` being the current one.
+
+### --log-file-compress
+
+If set, compress rotated log files using gzip. This changes the
+extension of the old log files to `.log.gz`.
+
+Defaults to false - don't compress log files.
+
+### --log-file-max-age Duration
+
+Maximum duration to retain old log files (eg `7d`). This is rounded to
+the dearest day, or 1 day if lower.
+
+The default is to retain all old log files.
+
+### --log-file-max-backups int
+
+Maximum number of old log files to retain
+
+The default is to retain all old log files.
+
### --log-format string
Comma separated list of log format options. The accepted options are:
diff --git a/fs/log/log.go b/fs/log/log.go
index 8e56a2d18..0c0fad727 100644
--- a/fs/log/log.go
+++ b/fs/log/log.go
@@ -9,8 +9,10 @@ import (
"reflect"
"runtime"
"strings"
+ "time"
"github.com/rclone/rclone/fs"
+ "gopkg.in/natefinch/lumberjack.v2"
)
// OptionsInfo descripts the Options in use
@@ -19,6 +21,26 @@ var OptionsInfo = fs.Options{{
Default: "",
Help: "Log everything to this file",
Groups: "Logging",
+}, {
+ Name: "log_file_max_size",
+ Default: fs.SizeSuffix(-1),
+ Help: `Maximum size of the log file before it's rotated (eg "10M")`,
+ Groups: "Logging",
+}, {
+ Name: "log_file_max_backups",
+ Default: 0,
+ Help: "Maximum number of old log files to retain.",
+ Groups: "Logging",
+}, {
+ Name: "log_file_max_age",
+ Default: fs.Duration(0),
+ Help: `Maximum duration to retain old log files (eg "7d")`,
+ Groups: "Logging",
+}, {
+ Name: "log_file_compress",
+ Default: false,
+ Help: "If set, compress rotated log files using gzip.",
+ Groups: "Logging",
}, {
Name: "log_format",
Default: logFormatDate | logFormatTime,
@@ -54,12 +76,16 @@ var OptionsInfo = fs.Options{{
// Options contains options for controlling the logging
type Options struct {
- File string `config:"log_file"` // Log everything to this file
- Format logFormat `config:"log_format"` // Comma separated list of log format options
- UseSyslog bool `config:"syslog"` // Use Syslog for logging
- SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,...
- LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging
- WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"`
+ File string `config:"log_file"` // Log everything to this file
+ MaxSize fs.SizeSuffix `config:"log_file_max_size"` // Max size of log file
+ MaxBackups int `config:"log_file_max_backups"` // Max backups of log file
+ MaxAge fs.Duration `config:"log_file_max_age"` // Max age of of log file
+ Compress bool `config:"log_file_compress"` // Set to compress log file
+ Format logFormat `config:"log_format"` // Comma separated list of log format options
+ UseSyslog bool `config:"syslog"` // Use Syslog for logging
+ SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,...
+ LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging
+ WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"`
}
func init() {
@@ -182,16 +208,42 @@ func InitLogging() {
// Log file output
if Opt.File != "" {
- f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
- if err != nil {
- fs.Fatalf(nil, "Failed to open log file: %v", err)
- }
- _, err = f.Seek(0, io.SeekEnd)
- if err != nil {
- fs.Errorf(nil, "Failed to seek log file to end: %v", err)
+ var w io.Writer
+ if Opt.MaxSize == 0 {
+ // No log rotation - just open the file as normal
+ // We'll capture tracebacks like this too.
+ f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
+ if err != nil {
+ fs.Fatalf(nil, "Failed to open log file: %v", err)
+ }
+ _, err = f.Seek(0, io.SeekEnd)
+ if err != nil {
+ fs.Errorf(nil, "Failed to seek log file to end: %v", err)
+ }
+ redirectStderr(f)
+ w = f
+ } else {
+ // Round with a minimum of 1 if set
+ round := func(x float64) int {
+ if x <= 0 {
+ return 0
+ } else if x <= 1 {
+ return 1
+ }
+ return int(x + 0.5)
+ }
+ // Log rotation active
+ f := &lumberjack.Logger{
+ Filename: Opt.File,
+ MaxSize: round(float64(Opt.MaxSize) / float64(fs.Mebi)), // MiB
+ MaxBackups: Opt.MaxBackups,
+ MaxAge: round(time.Duration(Opt.MaxAge).Hours() / 24), // Days
+ Compress: Opt.Compress,
+ LocalTime: true, // format log file names in localtime
+ }
+ w = f
}
- redirectStderr(f)
- Handler.setWriter(f)
+ Handler.setWriter(w)
}
// --use-json-log implies JSON formatting
diff --git a/go.mod b/go.mod
index 72e0bb01f..106a293a8 100644
--- a/go.mod
+++ b/go.mod
@@ -89,6 +89,7 @@ require (
golang.org/x/text v0.28.0
golang.org/x/time v0.12.0
google.golang.org/api v0.247.0
+ gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/validator.v2 v2.0.1
gopkg.in/yaml.v3 v3.0.1
storj.io/uplink v1.13.1
diff --git a/go.sum b/go.sum
index c7c752ec1..f4ce2432b 100644
--- a/go.sum
+++ b/go.sum
@@ -1045,6 +1045,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=