Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

rc: add sync report parameters to sync/sync, sync/copy and sync/move - fixes #9846

Nick Craig-Woodaf382608c4a7
diff --git a/fs/operations/logger.go b/fs/operations/logger.go
index 0d482206f..f9fb41f0e 100644
--- a/fs/operations/logger.go
+++ b/fs/operations/logger.go
@@ -302,12 +302,54 @@ func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err err
return winner
}
+// NewSyncLoggerOpt returns a LoggerOpt with no report writers set and
+// the default listing options for the DestAfter report.
+func NewSyncLoggerOpt() LoggerOpt {
+ return LoggerOpt{
+ Format: "p",
+ Separator: ";",
+ DirSlash: true,
+ HashType: hash.MD5,
+ FilesOnly: true,
+ }
+}
+
+// Init prepares opt for logging a sync to fdst once its report writers
+// have been set.
+//
+// It configures the DestAfter listing from the lsf options and warns
+// about reports which --no-traverse prevents being complete. cmdFlags
+// may be nil if the options did not come from command line flags.
+func (opt *LoggerOpt) Init(ctx context.Context, fdst fs.Fs, cmdFlags *pflag.FlagSet) {
+ if opt.TimeFormat == "max" {
+ opt.TimeFormat = FormatForLSFPrecision(fdst.Precision())
+ }
+ opt.SetListFormat(ctx, cmdFlags)
+ opt.NewListJSON(ctx, fdst, "")
+
+ ci := fs.GetConfig(ctx)
+ if ci.NoTraverse && opt.Combined != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
+ }
+ if ci.NoTraverse && opt.MissingOnSrc != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
+ }
+ if ci.NoTraverse && opt.DestAfter != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
+ }
+}
+
// SetListFormat sets opt.ListFormat for destAfter
+//
+// cmdFlags may be nil if the options did not come from command line flags.
// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from
func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {
// Work out if the separatorFlag was supplied or not
- separatorFlag := cmdFlags.Lookup("separator")
- separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+ separatorFlagSupplied := false
+ if cmdFlags != nil {
+ separatorFlag := cmdFlags.Lookup("separator")
+ separatorFlagSupplied = separatorFlag != nil && separatorFlag.Changed
+ }
// Default the separator to , if using CSV
if opt.Csv && !separatorFlagSupplied {
opt.Separator = ","
diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go
index 054ed96bc..e2cfda034 100644
--- a/fs/operations/operationsflags/operationsflags.go
+++ b/fs/operations/operationsflags/operationsflags.go
@@ -11,7 +11,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
- "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@@ -61,16 +60,17 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")
// lsf flags for destAfter
- flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
- flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")
- flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
- flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
- opt.HashType = hash.MD5
+ def := operations.NewSyncLoggerOpt()
+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", def.Format, "Output format - see lsf help for details", "Sync")
+ flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", def.TimeFormat, "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")
+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", def.Separator, "Separator for the items in the format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", def.DirSlash, "Append a slash to directory names", "Sync")
+ opt.HashType = def.HashType
flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
- flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
- flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
- flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
- flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", def.FilesOnly, "Only list files", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", def.DirsOnly, "Only list directories", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", def.Csv, "Output in CSV format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", def.Absolute, "Put a leading / in front of path names", "Sync")
// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
}
@@ -78,12 +78,6 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
closers := []io.Closer{}
- if opt.TimeFormat == "max" {
- opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
- }
- opt.SetListFormat(ctx, command.Flags())
- opt.NewListJSON(ctx, fdst, "")
-
open := func(name string, pout *io.Writer) error {
if name == "" {
return nil
@@ -132,16 +126,7 @@ func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, o
}
}
- ci := fs.GetConfig(ctx)
- if ci.NoTraverse && opt.Combined != nil {
- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
- }
- if ci.NoTraverse && opt.MissingOnSrc != nil {
- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
- }
- if ci.NoTraverse && opt.DestAfter != nil {
- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
- }
+ opt.Init(ctx, fdst, command.Flags())
return close, nil
}
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index 27a8d5c9e..4c0828127 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -764,6 +764,25 @@ func (s stringWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
+// RcReportWriter returns a writer for the report called name.
+//
+// The report is enabled if in[name] is true, or if in[name] is absent
+// and Default is true. When enabled, each line written to the returned
+// writer is collected as a string in out[name], otherwise nil is
+// returned to disable the report.
+func RcReportWriter(in rc.Params, out rc.Params, name string, Default bool) io.Writer {
+ active, err := in.GetBool(name)
+ if err != nil {
+ active = Default
+ }
+ if !active {
+ return nil
+ }
+ result := []string{}
+ out[name] = &result
+ return stringWriter{&result}
+}
+
// Check two directories
func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")
@@ -822,26 +841,12 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
}
out = rc.Params{}
-
- getOutput := func(name string, Default bool) io.Writer {
- active, err := in.GetBool(name)
- if err != nil {
- active = Default
- }
- if !active {
- return nil
- }
- result := []string{}
- out[name] = &result
- return stringWriter{&result}
- }
-
- opt.Combined = getOutput("combined", false)
- opt.MissingOnSrc = getOutput("missingOnSrc", true)
- opt.MissingOnDst = getOutput("missingOnDst", true)
- opt.Match = getOutput("match", false)
- opt.Differ = getOutput("differ", true)
- opt.Error = getOutput("error", true)
+ opt.Combined = RcReportWriter(in, out, "combined", false)
+ opt.MissingOnSrc = RcReportWriter(in, out, "missingOnSrc", true)
+ opt.MissingOnDst = RcReportWriter(in, out, "missingOnDst", true)
+ opt.Match = RcReportWriter(in, out, "match", false)
+ opt.Differ = RcReportWriter(in, out, "differ", true)
+ opt.Error = RcReportWriter(in, out, "error", true)
if checkFileHash != "" {
out["hashType"] = checkFileHashType.String()
diff --git a/fs/sync/rc.go b/fs/sync/rc.go
index 52a2626e8..fe2b67021 100644
--- a/fs/sync/rc.go
+++ b/fs/sync/rc.go
@@ -3,6 +3,8 @@ package sync
import (
"context"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/rc"
)
@@ -23,7 +25,27 @@ func init() {
- srcFs - a remote name string e.g. "drive:src" for the source
- dstFs - a remote name string e.g. "drive:dst" for the destination
- createEmptySrcDirs - create empty src directories on destination if set
-` + moveHelp + `
+` + moveHelp + `- combined - make a combined report of changes (default false)
+- missingOnSrc - report all files missing from the source (default false)
+- missingOnDst - report all files missing from the destination (default false)
+- match - report all matching files (default false)
+- differ - report all non-matching files (default false)
+- error - report all files with errors (hashing or reading) (default false)
+- destAfter - report all files that exist on the destination post-` + name + ` (default false)
+
+Returns:
+
+- combined - array of strings of combined report of changes
+- missingOnSrc - array of strings of all files missing from the source
+- missingOnDst - array of strings of all files missing from the destination
+- match - array of strings of all matching files
+- differ - array of strings of all non-matching files
+- error - array of strings of all files with errors (hashing or reading)
+- destAfter - array of strings of all files that exist on the destination post-` + name + `
+
+Each report is only returned if its parameter is set to true. If the
+operation fails the reports are only available if it was run with
+` + "`_async`" + `, as part of the job output.
See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`,
})
@@ -44,17 +66,38 @@ func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Para
if rc.NotErrParamNotFound(err) {
return nil, err
}
+ ctx, out = rcLogger(ctx, in, dstFs)
switch name {
case "sync":
- return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)
+ return out, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)
case "copy":
- return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)
+ return out, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)
case "move":
deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs")
if rc.NotErrParamNotFound(err) {
return nil, err
}
- return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
+ return out, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
}
panic("unknown rcSyncCopyMove type")
}
+
+// rcLogger returns ctx with a sync logger which collects the reports
+// requested in in into the returned out, which is empty if none were.
+func rcLogger(ctx context.Context, in rc.Params, fdst fs.Fs) (context.Context, rc.Params) {
+ out := rc.Params{}
+ opt := operations.NewSyncLoggerOpt()
+ opt.Combined = operations.RcReportWriter(in, out, "combined", false)
+ opt.MissingOnSrc = operations.RcReportWriter(in, out, "missingOnSrc", false)
+ opt.MissingOnDst = operations.RcReportWriter(in, out, "missingOnDst", false)
+ opt.Match = operations.RcReportWriter(in, out, "match", false)
+ opt.Differ = operations.RcReportWriter(in, out, "differ", false)
+ opt.Error = operations.RcReportWriter(in, out, "error", false)
+ opt.DestAfter = operations.RcReportWriter(in, out, "destAfter", false)
+ if len(out) == 0 {
+ return ctx, out
+ }
+ opt.LoggerFn = operations.NewDefaultLoggerFn(&opt)
+ opt.Init(ctx, fdst, nil)
+ return operations.WithSyncLogger(ctx, opt), out
+}
diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.go
index ad5145633..aeaae4bcd 100644
--- a/fs/sync/rc_test.go
+++ b/fs/sync/rc_test.go
@@ -2,6 +2,7 @@ package sync
import (
"context"
+ "sort"
"testing"
"github.com/rclone/rclone/fs/cache"
@@ -41,7 +42,7 @@ func TestRcCopy(t *testing.T) {
}
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
- assert.Equal(t, rc.Params(nil), out)
+ assert.Equal(t, rc.Params{}, out)
r.CheckLocalItems(t, file1, file2)
r.CheckRemoteItems(t, file1, file2, file3)
@@ -65,7 +66,7 @@ func TestRcMove(t *testing.T) {
}
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
- assert.Equal(t, rc.Params(nil), out)
+ assert.Equal(t, rc.Params{}, out)
r.CheckLocalItems(t)
r.CheckRemoteItems(t, file1, file2, file3)
@@ -89,8 +90,53 @@ func TestRcSync(t *testing.T) {
}
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
- assert.Equal(t, rc.Params(nil), out)
+ assert.Equal(t, rc.Params{}, out)
r.CheckLocalItems(t, file1, file2)
r.CheckRemoteItems(t, file1, file2)
}
+
+// sync/copy: check the reports are returned when requested
+func TestRcCopyReports(t *testing.T) {
+ r, call := rcNewRun(t, "sync/copy")
+ r.Mkdir(context.Background(), r.Fremote)
+
+ file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
+ file2 := r.WriteFile("subdir/file2", "file2 contents", t2)
+ file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)
+ file4 := r.WriteFile("file4", "file4 contents", t1)
+ file4dst := r.WriteObject(context.Background(), "file4", "different contents", t1)
+
+ r.CheckLocalItems(t, file1, file2, file4)
+ r.CheckRemoteItems(t, file1, file3, file4dst)
+
+ in := rc.Params{
+ "srcFs": r.LocalName,
+ "dstFs": r.FremoteName,
+ "combined": true,
+ "missingOnSrc": true,
+ "missingOnDst": true,
+ "match": true,
+ "differ": true,
+ "destAfter": true,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+
+ sorted := func(name string) []string {
+ result, ok := out[name].(*[]string)
+ require.True(t, ok, name)
+ sort.Strings(*result)
+ return *result
+ }
+ assert.Equal(t, []string{"* file4", "+ subdir/file2", "- subdir/subsubdir/file3", "= file1"}, sorted("combined"))
+ assert.Equal(t, []string{"subdir/subsubdir/file3"}, sorted("missingOnSrc"))
+ assert.Equal(t, []string{"subdir/file2"}, sorted("missingOnDst"))
+ assert.Equal(t, []string{"file1"}, sorted("match"))
+ assert.Equal(t, []string{"file4"}, sorted("differ"))
+ assert.Equal(t, []string{"file1", "file4", "subdir/file2", "subdir/subsubdir/file3"}, sorted("destAfter"))
+ assert.NotContains(t, out, "error")
+
+ r.CheckLocalItems(t, file1, file2, file4)
+ r.CheckRemoteItems(t, file1, file2, file3, file4)
+}