"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
rc: add sync report parameters to sync/sync, sync/copy and sync/move - fixes #9846
Nick Craig-Woodaf382608c4a7
diff --git a/fs/operations/logger.go b/fs/operations/logger.goindex 0d482206f..f9fb41f0e 100644--- a/fs/operations/logger.go+++ b/fs/operations/logger.go@@ -302,12 +302,54 @@ func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err errreturn winner}+// NewSyncLoggerOpt returns a LoggerOpt with no report writers set and+// the default listing options for the DestAfter report.+func NewSyncLoggerOpt() LoggerOpt {+ return LoggerOpt{+ Format: "p",+ Separator: ";",+ DirSlash: true,+ HashType: hash.MD5,+ FilesOnly: true,+ }+}++// Init prepares opt for logging a sync to fdst once its report writers+// have been set.+//+// It configures the DestAfter listing from the lsf options and warns+// about reports which --no-traverse prevents being complete. cmdFlags+// may be nil if the options did not come from command line flags.+func (opt *LoggerOpt) Init(ctx context.Context, fdst fs.Fs, cmdFlags *pflag.FlagSet) {+ if opt.TimeFormat == "max" {+ opt.TimeFormat = FormatForLSFPrecision(fdst.Precision())+ }+ opt.SetListFormat(ctx, cmdFlags)+ opt.NewListJSON(ctx, fdst, "")++ ci := fs.GetConfig(ctx)+ if ci.NoTraverse && opt.Combined != nil {+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")+ }+ if ci.NoTraverse && opt.MissingOnSrc != nil {+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")+ }+ if ci.NoTraverse && opt.DestAfter != nil {+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")+ }+}+// SetListFormat sets opt.ListFormat for destAfter+//+// cmdFlags may be nil if the options did not come from command line flags.// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied fromfunc (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {// Work out if the separatorFlag was supplied or not- separatorFlag := cmdFlags.Lookup("separator")- separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed+ separatorFlagSupplied := false+ if cmdFlags != nil {+ separatorFlag := cmdFlags.Lookup("separator")+ separatorFlagSupplied = separatorFlag != nil && separatorFlag.Changed+ }// Default the separator to , if using CSVif opt.Csv && !separatorFlagSupplied {opt.Separator = ","diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.goindex 054ed96bc..e2cfda034 100644--- a/fs/operations/operationsflags/operationsflags.go+++ b/fs/operations/operationsflags/operationsflags.go@@ -11,7 +11,6 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/flags"- "github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""github.com/spf13/cobra""github.com/spf13/pflag"@@ -61,16 +60,17 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOptflags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")// lsf flags for destAfter- flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")- flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")- flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")- flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")- opt.HashType = hash.MD5+ def := operations.NewSyncLoggerOpt()+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", def.Format, "Output format - see lsf help for details", "Sync")+ flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", def.TimeFormat, "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", def.Separator, "Separator for the items in the format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", def.DirSlash, "Append a slash to directory names", "Sync")+ opt.HashType = def.HashTypeflags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")- flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")- flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")- flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")- flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", def.FilesOnly, "Only list files", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", def.DirsOnly, "Only list directories", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", def.Csv, "Output in CSV format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", def.Absolute, "Put a leading / in front of path names", "Sync")// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")}@@ -78,12 +78,6 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOptfunc ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {closers := []io.Closer{}- if opt.TimeFormat == "max" {- opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())- }- opt.SetListFormat(ctx, command.Flags())- opt.NewListJSON(ctx, fdst, "")-open := func(name string, pout *io.Writer) error {if name == "" {return nil@@ -132,16 +126,7 @@ func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, o}}- ci := fs.GetConfig(ctx)- if ci.NoTraverse && opt.Combined != nil {- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")- }- if ci.NoTraverse && opt.MissingOnSrc != nil {- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")- }- if ci.NoTraverse && opt.DestAfter != nil {- fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")- }+ opt.Init(ctx, fdst, command.Flags())return close, nil}diff --git a/fs/operations/rc.go b/fs/operations/rc.goindex 27a8d5c9e..4c0828127 100644--- a/fs/operations/rc.go+++ b/fs/operations/rc.go@@ -764,6 +764,25 @@ func (s stringWriter) Write(p []byte) (n int, err error) {return len(p), nil}+// RcReportWriter returns a writer for the report called name.+//+// The report is enabled if in[name] is true, or if in[name] is absent+// and Default is true. When enabled, each line written to the returned+// writer is collected as a string in out[name], otherwise nil is+// returned to disable the report.+func RcReportWriter(in rc.Params, out rc.Params, name string, Default bool) io.Writer {+ active, err := in.GetBool(name)+ if err != nil {+ active = Default+ }+ if !active {+ return nil+ }+ result := []string{}+ out[name] = &result+ return stringWriter{&result}+}+// Check two directoriesfunc rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")@@ -822,26 +841,12 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {}out = rc.Params{}-- getOutput := func(name string, Default bool) io.Writer {- active, err := in.GetBool(name)- if err != nil {- active = Default- }- if !active {- return nil- }- result := []string{}- out[name] = &result- return stringWriter{&result}- }-- opt.Combined = getOutput("combined", false)- opt.MissingOnSrc = getOutput("missingOnSrc", true)- opt.MissingOnDst = getOutput("missingOnDst", true)- opt.Match = getOutput("match", false)- opt.Differ = getOutput("differ", true)- opt.Error = getOutput("error", true)+ opt.Combined = RcReportWriter(in, out, "combined", false)+ opt.MissingOnSrc = RcReportWriter(in, out, "missingOnSrc", true)+ opt.MissingOnDst = RcReportWriter(in, out, "missingOnDst", true)+ opt.Match = RcReportWriter(in, out, "match", false)+ opt.Differ = RcReportWriter(in, out, "differ", true)+ opt.Error = RcReportWriter(in, out, "error", true)if checkFileHash != "" {out["hashType"] = checkFileHashType.String()diff --git a/fs/sync/rc.go b/fs/sync/rc.goindex 52a2626e8..fe2b67021 100644--- a/fs/sync/rc.go+++ b/fs/sync/rc.go@@ -3,6 +3,8 @@ package syncimport ("context"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/rc")@@ -23,7 +25,27 @@ func init() {- srcFs - a remote name string e.g. "drive:src" for the source- dstFs - a remote name string e.g. "drive:dst" for the destination- createEmptySrcDirs - create empty src directories on destination if set-` + moveHelp + `+` + moveHelp + `- combined - make a combined report of changes (default false)+- missingOnSrc - report all files missing from the source (default false)+- missingOnDst - report all files missing from the destination (default false)+- match - report all matching files (default false)+- differ - report all non-matching files (default false)+- error - report all files with errors (hashing or reading) (default false)+- destAfter - report all files that exist on the destination post-` + name + ` (default false)++Returns:++- combined - array of strings of combined report of changes+- missingOnSrc - array of strings of all files missing from the source+- missingOnDst - array of strings of all files missing from the destination+- match - array of strings of all matching files+- differ - array of strings of all non-matching files+- error - array of strings of all files with errors (hashing or reading)+- destAfter - array of strings of all files that exist on the destination post-` + name + `++Each report is only returned if its parameter is set to true. If the+operation fails the reports are only available if it was run with+` + "`_async`" + `, as part of the job output.See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`,})@@ -44,17 +66,38 @@ func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Paraif rc.NotErrParamNotFound(err) {return nil, err}+ ctx, out = rcLogger(ctx, in, dstFs)switch name {case "sync":- return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)+ return out, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)case "copy":- return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)+ return out, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)case "move":deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs")if rc.NotErrParamNotFound(err) {return nil, err}- return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)+ return out, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)}panic("unknown rcSyncCopyMove type")}++// rcLogger returns ctx with a sync logger which collects the reports+// requested in in into the returned out, which is empty if none were.+func rcLogger(ctx context.Context, in rc.Params, fdst fs.Fs) (context.Context, rc.Params) {+ out := rc.Params{}+ opt := operations.NewSyncLoggerOpt()+ opt.Combined = operations.RcReportWriter(in, out, "combined", false)+ opt.MissingOnSrc = operations.RcReportWriter(in, out, "missingOnSrc", false)+ opt.MissingOnDst = operations.RcReportWriter(in, out, "missingOnDst", false)+ opt.Match = operations.RcReportWriter(in, out, "match", false)+ opt.Differ = operations.RcReportWriter(in, out, "differ", false)+ opt.Error = operations.RcReportWriter(in, out, "error", false)+ opt.DestAfter = operations.RcReportWriter(in, out, "destAfter", false)+ if len(out) == 0 {+ return ctx, out+ }+ opt.LoggerFn = operations.NewDefaultLoggerFn(&opt)+ opt.Init(ctx, fdst, nil)+ return operations.WithSyncLogger(ctx, opt), out+}diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.goindex ad5145633..aeaae4bcd 100644--- a/fs/sync/rc_test.go+++ b/fs/sync/rc_test.go@@ -2,6 +2,7 @@ package syncimport ("context"+ "sort""testing""github.com/rclone/rclone/fs/cache"@@ -41,7 +42,7 @@ func TestRcCopy(t *testing.T) {}out, err := call.Fn(context.Background(), in)require.NoError(t, err)- assert.Equal(t, rc.Params(nil), out)+ assert.Equal(t, rc.Params{}, out)r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t, file1, file2, file3)@@ -65,7 +66,7 @@ func TestRcMove(t *testing.T) {}out, err := call.Fn(context.Background(), in)require.NoError(t, err)- assert.Equal(t, rc.Params(nil), out)+ assert.Equal(t, rc.Params{}, out)r.CheckLocalItems(t)r.CheckRemoteItems(t, file1, file2, file3)@@ -89,8 +90,53 @@ func TestRcSync(t *testing.T) {}out, err := call.Fn(context.Background(), in)require.NoError(t, err)- assert.Equal(t, rc.Params(nil), out)+ assert.Equal(t, rc.Params{}, out)r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t, file1, file2)}++// sync/copy: check the reports are returned when requested+func TestRcCopyReports(t *testing.T) {+ r, call := rcNewRun(t, "sync/copy")+ r.Mkdir(context.Background(), r.Fremote)++ file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)+ file2 := r.WriteFile("subdir/file2", "file2 contents", t2)+ file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)+ file4 := r.WriteFile("file4", "file4 contents", t1)+ file4dst := r.WriteObject(context.Background(), "file4", "different contents", t1)++ r.CheckLocalItems(t, file1, file2, file4)+ r.CheckRemoteItems(t, file1, file3, file4dst)++ in := rc.Params{+ "srcFs": r.LocalName,+ "dstFs": r.FremoteName,+ "combined": true,+ "missingOnSrc": true,+ "missingOnDst": true,+ "match": true,+ "differ": true,+ "destAfter": true,+ }+ out, err := call.Fn(context.Background(), in)+ require.NoError(t, err)++ sorted := func(name string) []string {+ result, ok := out[name].(*[]string)+ require.True(t, ok, name)+ sort.Strings(*result)+ return *result+ }+ assert.Equal(t, []string{"* file4", "+ subdir/file2", "- subdir/subsubdir/file3", "= file1"}, sorted("combined"))+ assert.Equal(t, []string{"subdir/subsubdir/file3"}, sorted("missingOnSrc"))+ assert.Equal(t, []string{"subdir/file2"}, sorted("missingOnDst"))+ assert.Equal(t, []string{"file1"}, sorted("match"))+ assert.Equal(t, []string{"file4"}, sorted("differ"))+ assert.Equal(t, []string{"file1", "file4", "subdir/file2", "subdir/subsubdir/file3"}, sorted("destAfter"))+ assert.NotContains(t, out, "error")++ r.CheckLocalItems(t, file1, file2, file4)+ r.CheckRemoteItems(t, file1, file2, file3, file4)+}