"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
log: add log rotation to --log-file - fixes #2259
Nick Craig-Wood8d353039a694
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex de5b9590d..ba9435781 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -1727,6 +1727,57 @@ Note that if you are using the `logrotate` program to manage rclone'slogs, then you should use the `copytruncate` option as rclone doesn'thave a signal to rotate logs.+Alternatively you can use the options below to manage rclone's built+in log rotation.++### --log-file-max-size SizeSuffix++Maximum size of the log file before it's rotated (eg `10M`). This SizeSuffix+is rounded to the nearest MiB or 1 MiB if lower.++If `--log-file` is not set then this option will be ignored.++If this option is not set, then the other log rotation options will be+ignored.++For example if the following flags are in use++```sh+rclone --log-file rclone.log --log-file-max-size 1M --log-file-max-backups 3+```++Then this will create log files which look like this++```console+$ ls -l+-rw------- 1 user user 1048491 Apr 11 17:15 rclone-2025-04-11T17-15-29.998.log+-rw------- 1 user user 1048511 Apr 11 17:15 rclone-2025-04-11T17-15-30.467.log+-rw------- 1 user user 1048559 Apr 11 17:15 rclone-2025-04-11T17-15-30.543.log+-rw------- 1 user user 521602 Apr 11 17:15 rclone.log+```++The file `rclone.log` being the current one.++### --log-file-compress++If set, compress rotated log files using gzip. This changes the+extension of the old log files to `.log.gz`.++Defaults to false - don't compress log files.++### --log-file-max-age Duration++Maximum duration to retain old log files (eg `7d`). This is rounded to+the dearest day, or 1 day if lower.++The default is to retain all old log files.++### --log-file-max-backups int++Maximum number of old log files to retain++The default is to retain all old log files.+### --log-format stringComma separated list of log format options. The accepted options are:diff --git a/fs/log/log.go b/fs/log/log.goindex 8e56a2d18..0c0fad727 100644--- a/fs/log/log.go+++ b/fs/log/log.go@@ -9,8 +9,10 @@ import ("reflect""runtime""strings"+ "time""github.com/rclone/rclone/fs"+ "gopkg.in/natefinch/lumberjack.v2")// OptionsInfo descripts the Options in use@@ -19,6 +21,26 @@ var OptionsInfo = fs.Options{{Default: "",Help: "Log everything to this file",Groups: "Logging",+}, {+ Name: "log_file_max_size",+ Default: fs.SizeSuffix(-1),+ Help: `Maximum size of the log file before it's rotated (eg "10M")`,+ Groups: "Logging",+}, {+ Name: "log_file_max_backups",+ Default: 0,+ Help: "Maximum number of old log files to retain.",+ Groups: "Logging",+}, {+ Name: "log_file_max_age",+ Default: fs.Duration(0),+ Help: `Maximum duration to retain old log files (eg "7d")`,+ Groups: "Logging",+}, {+ Name: "log_file_compress",+ Default: false,+ Help: "If set, compress rotated log files using gzip.",+ Groups: "Logging",}, {Name: "log_format",Default: logFormatDate | logFormatTime,@@ -54,12 +76,16 @@ var OptionsInfo = fs.Options{{// Options contains options for controlling the loggingtype Options struct {- File string `config:"log_file"` // Log everything to this file- Format logFormat `config:"log_format"` // Comma separated list of log format options- UseSyslog bool `config:"syslog"` // Use Syslog for logging- SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,...- LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging- WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"`+ File string `config:"log_file"` // Log everything to this file+ MaxSize fs.SizeSuffix `config:"log_file_max_size"` // Max size of log file+ MaxBackups int `config:"log_file_max_backups"` // Max backups of log file+ MaxAge fs.Duration `config:"log_file_max_age"` // Max age of of log file+ Compress bool `config:"log_file_compress"` // Set to compress log file+ Format logFormat `config:"log_format"` // Comma separated list of log format options+ UseSyslog bool `config:"syslog"` // Use Syslog for logging+ SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,...+ LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging+ WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"`}func init() {@@ -182,16 +208,42 @@ func InitLogging() {// Log file outputif Opt.File != "" {- f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)- if err != nil {- fs.Fatalf(nil, "Failed to open log file: %v", err)- }- _, err = f.Seek(0, io.SeekEnd)- if err != nil {- fs.Errorf(nil, "Failed to seek log file to end: %v", err)+ var w io.Writer+ if Opt.MaxSize == 0 {+ // No log rotation - just open the file as normal+ // We'll capture tracebacks like this too.+ f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)+ if err != nil {+ fs.Fatalf(nil, "Failed to open log file: %v", err)+ }+ _, err = f.Seek(0, io.SeekEnd)+ if err != nil {+ fs.Errorf(nil, "Failed to seek log file to end: %v", err)+ }+ redirectStderr(f)+ w = f+ } else {+ // Round with a minimum of 1 if set+ round := func(x float64) int {+ if x <= 0 {+ return 0+ } else if x <= 1 {+ return 1+ }+ return int(x + 0.5)+ }+ // Log rotation active+ f := &lumberjack.Logger{+ Filename: Opt.File,+ MaxSize: round(float64(Opt.MaxSize) / float64(fs.Mebi)), // MiB+ MaxBackups: Opt.MaxBackups,+ MaxAge: round(time.Duration(Opt.MaxAge).Hours() / 24), // Days+ Compress: Opt.Compress,+ LocalTime: true, // format log file names in localtime+ }+ w = f}- redirectStderr(f)- Handler.setWriter(f)+ Handler.setWriter(w)}// --use-json-log implies JSON formattingdiff --git a/go.mod b/go.modindex 72e0bb01f..106a293a8 100644--- a/go.mod+++ b/go.mod@@ -89,6 +89,7 @@ require (golang.org/x/text v0.28.0golang.org/x/time v0.12.0google.golang.org/api v0.247.0+ gopkg.in/natefinch/lumberjack.v2 v2.2.1gopkg.in/validator.v2 v2.0.1gopkg.in/yaml.v3 v3.0.1storj.io/uplink v1.13.1diff --git a/go.sum b/go.sumindex c7c752ec1..f4ce2432b 100644--- a/go.sum+++ b/go.sum@@ -1045,6 +1045,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNgopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=