"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
sync: report list of synced paths to file -- see #7282
nielash3a50f35df9ff
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.goindex da9184f30..38076541e 100644--- a/cmd/sync/sync.go+++ b/cmd/sync/sync.go@@ -3,22 +3,137 @@ package syncimport ("context"+ "io"+ "os"++ mutex "sync" // renamed as "sync" already in use"github.com/rclone/rclone/cmd"+ "github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/flags""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/operations/operationsflags""github.com/rclone/rclone/fs/sync""github.com/spf13/cobra")var (createEmptySrcDirs = false+ opt = operations.LoggerOpt{}+ loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{})func init() {cmd.Root.AddCommand(commandDefinition)cmdFlags := commandDefinition.Flags()flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")+ operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)+ // TODO: add same flags to move and copy+}++var lock mutex.Mutex++func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {+ lock.Lock()+ defer lock.Unlock()++ if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {+ opt.PrintDestAfter(ctx, sigil, src, dst, err)+ return+ }++ _, srcOk := src.(fs.Object)+ _, dstOk := dst.(fs.Object)+ var filename string+ if !srcOk && !dstOk {+ return+ } else if srcOk && !dstOk {+ filename = src.String()+ } else {+ filename = dst.String()+ }++ if sigil.Writer(opt) != nil {+ operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)+ }+ if opt.Combined != nil {+ operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)+ fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)+ }+ if opt.DestAfter != nil {+ opt.PrintDestAfter(ctx, sigil, src, dst, err)+ }+}++// GetSyncLoggerOpt gets the options corresponding to the logger flags+func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {+ closers := []io.Closer{}++ opt.LoggerFn = syncLoggerFn+ if opt.TimeFormat == "max" {+ opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())+ }+ opt.SetListFormat(ctx, command.Flags())+ opt.NewListJSON(ctx, fdst, "")++ open := func(name string, pout *io.Writer) error {+ if name == "" {+ return nil+ }+ if name == "-" {+ *pout = os.Stdout+ return nil+ }+ out, err := os.Create(name)+ if err != nil {+ return err+ }+ *pout = out+ closers = append(closers, out)+ return nil+ }++ if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {+ return opt, nil, err+ }+ if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {+ return opt, nil, err+ }++ close := func() {+ for _, closer := range closers {+ err := closer.Close()+ if err != nil {+ fs.Errorf(nil, "Failed to close report output: %v", err)+ }+ }+ }++ return opt, close, nil+}++func anyNotBlank(s ...string) bool {+ for _, x := range s {+ if x != "" {+ return true+ }+ }+ return false}var commandDefinition = &cobra.Command{@@ -59,6 +174,44 @@ destination that is inside the source directory.**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.++## Logger Flags++The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` ++ "`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or+stdout if it is ` + "`-`" + `) supplied. What they write is described in the+help below. For example ` + "`--differ`" + ` will write all paths which are+present on both the source and destination but different.++The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all+file paths with a symbol and then a space and then the path to tell+you what happened to it. These are reminiscent of diff files.++- ` + "`= path`" + ` means path was found in source and destination and was identical+- ` + "`- path`" + ` means path was missing on the source, so only in the destination+- ` + "`+ path`" + ` means path was missing on the destination, so only in the source+- ` + "`* path`" + ` means path was present in source and destination but different.+- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.++The ` + "`--dest-after`" + ` flag writes a list file using the same format flags+as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))+Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical+-- it should output an accurate list of what will be on the destination+after the sync.++Note that these logger flags have a few limitations, and certain scenarios+are not currently supported:++- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `+- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `+- server-side moves of an entire dir at once+- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)+- Possibly some unusual error scenarios++Note also that each file is logged during the sync, as opposed to after, so it+is most useful as a predictor of what SHOULD happen to each file+(which may or may not match what actually DID.)`,Annotations: map[string]string{"groups": "Sync,Copy,Filter,Listing,Important",@@ -67,10 +220,22 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14cmd.CheckArgs(2, 2, command, args)fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)cmd.Run(true, true, command, func() error {+ ctx := context.Background()+ opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)+ if err != nil {+ return err+ }+ defer close()++ if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,+ loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {+ ctx = operations.WithSyncLogger(ctx, opt)+ }+if srcFileName == "" {- return sync.Sync(context.Background(), fdst, fsrc, createEmptySrcDirs)+ return sync.Sync(ctx, fdst, fsrc, createEmptySrcDirs)}- return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)+ return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)})},}diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.mdindex 8096e01c3..5ecbb3c03 100644--- a/docs/content/commands/rclone_sync.md+++ b/docs/content/commands/rclone_sync.md@@ -48,6 +48,43 @@ destination that is inside the source directory.**Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.+## Logger Flags++The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`+and `--error` flags write paths, one per line, to the file name (or+stdout if it is `-`) supplied. What they write is described in the+help below. For example `--differ` will write all paths which are+present on both the source and destination but different.++The `--combined` flag will write a file (or stdout) which contains all+file paths with a symbol and then a space and then the path to tell+you what happened to it. These are reminiscent of diff files.++- `= path` means path was found in source and destination and was identical+- `- path` means path was missing on the source, so only in the destination+- `+ path` means path was missing on the destination, so only in the source+- `* path` means path was present in source and destination but different.+- `! path` means there was an error reading or hashing the source or dest.++The `--dest-after` flag writes a list file using the same format flags+as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))+Conceptually it is similar to rsync's `--itemize-changes`, but not identical+-- it should output an accurate list of what will be on the destination+after the sync.++Note that these logger flags have a few limitations, and certain scenarios+are not currently supported:++- `--max-duration` / `CutoffModeHard`+- `--compare-dest` / `--copy-dest`+- server-side moves of an entire dir at once+- High-level retries, because there would be duplicates (use `--retries 1` to disable)+- Possibly some unusual error scenarios++Note also that each file is logged during the sync, as opposed to after, so it+is most useful as a predictor of what SHOULD happen to each file+(which may or may not match what actually DID.)```rclone sync source:path dest:path [flags]diff --git a/fs/logger/logger.go b/fs/logger/logger.gonew file mode 100644index 000000000..722b5d396--- /dev/null+++ b/fs/logger/logger.go@@ -0,0 +1,17 @@+// Package logger implements testing for the sync (and bisync) logger+package logger++import (+ _ "github.com/rclone/rclone/backend/all" // import all backends+ "github.com/rclone/rclone/cmd"+ _ "github.com/rclone/rclone/cmd/all" // import all commands+ _ "github.com/rclone/rclone/lib/plugin" // import plugins+)++// Main enables the testscript package. See:+// https://bitfieldconsulting.com/golang/cli-testing+// https://pkg.go.dev/github.com/rogpeppe/[email protected]/testscript+func Main() int {+ cmd.Main()+ return 0+}diff --git a/fs/logger/logger_test.go b/fs/logger/logger_test.gonew file mode 100644index 000000000..0e53563ce--- /dev/null+++ b/fs/logger/logger_test.go@@ -0,0 +1,33 @@+package logger_test++import (+ "os"+ "path/filepath"+ "testing"++ "github.com/rclone/rclone/fs/logger"+ "github.com/rogpeppe/go-internal/testscript"+)++// TestMain drives the tests+func TestMain(m *testing.M) {+ // This enables the testscript package. See:+ // https://bitfieldconsulting.com/golang/cli-testing+ // https://pkg.go.dev/github.com/rogpeppe/[email protected]/testscript+ os.Exit(testscript.RunMain(m, map[string]func() int{+ "rclone": logger.Main,+ }))+}++func TestLogger(t *testing.T) {+ // Usage: https://bitfieldconsulting.com/golang/cli-testing++ testscript.Run(t, testscript.Params{+ Dir: "testdata/script",+ Setup: func(env *testscript.Env) error {+ env.Setenv("SRC", filepath.Join("$WORK", "src"))+ env.Setenv("DST", filepath.Join("$WORK", "dst"))+ return nil+ },+ })+}diff --git a/fs/logger/testdata/script/TestBeforeVsAfter.txtar b/fs/logger/testdata/script/TestBeforeVsAfter.txtarnew file mode 100644index 000000000..0ccadc973--- /dev/null+++ b/fs/logger/testdata/script/TestBeforeVsAfter.txtar@@ -0,0 +1,48 @@+# tests whether an md5sum file generated post-sync matches our pre-sync prediction++# Filling src and dst with two different versions of rclone source!:+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip+exec unzip $SRC/src.zip -d $SRC+exec unzip $DST/dst.zip -d $DST+++# generating sumfiles:+exec rclone md5sum $SRC --output-file $WORK/src-before.txt+exec rclone md5sum $DST --output-file $WORK/dst-before.txt++# running sync with output files:+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt --dest-after $WORK/SYNCdestafter.txt --format 'hp' --separator ' '++# generating sumfiles:+exec rclone md5sum $SRC --output-file $WORK/src-after.txt+exec rclone md5sum $DST --output-file $WORK/dst-after.txt++# sorting them by line and diffing:+exec sort $WORK/src-before.txt -o $WORK/src-before.txt+exec sort $WORK/dst-before.txt -o $WORK/dst-before.txt+exec sort $WORK/src-after.txt -o $WORK/src-after.txt+exec sort $WORK/dst-after.txt -o $WORK/dst-after.txt++exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt+exec sort $WORK/SYNCdestafter.txt -o $WORK/SYNCdestafter.txt++# diff src before vs src after:+cmp $WORK/src-before.txt $WORK/src-after.txt++# diff dst before vs dst after:+! cmp $WORK/dst-before.txt $WORK/dst-after.txt++# diff src before vs dst after:+cmp $WORK/src-before.txt $WORK/dst-after.txt++# diff dst before vs src after:+! cmp $WORK/dst-before.txt $WORK/src-after.txt++# diff md5sum dst after vs sync dest-after:+cmp $WORK/dst-after.txt $WORK/SYNCdestafter.txt\ No newline at end of filediff --git a/fs/logger/testdata/script/TestCheckVsSync.txtar b/fs/logger/testdata/script/TestCheckVsSync.txtarnew file mode 100644index 000000000..11d2803d1--- /dev/null+++ b/fs/logger/testdata/script/TestCheckVsSync.txtar@@ -0,0 +1,48 @@+# tests whether rclone check and rclone sync output exactly the same file lists.++# Filling src and dst with makefiles:+exec rclone test makefiles $SRC --seed 0+exec rclone test makefiles $DST --seed 1+exec rclone test makefiles $SRC --seed 2+exec rclone test makefiles $DST --seed 2++# running rclone check for baseline test:+# error is expected here:+! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q++# running sync with output files:+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt++# sorting them by line and diffing:+exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt+exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt+exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt+exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt+exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt+exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt++exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt++# diff match check vs. sync:+cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt+# diff combined check vs. sync:+cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt+# diff missingonsrc check vs. sync:+cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt+# diff missingondst check vs. sync:+cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt+# diff error check vs. sync:+cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt+# diff differ check vs. sync:+cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt++# verify accuracy+exec rclone check $SRC $DST+exec diff -rya --suppress-common-lines $SRC $DST+[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst+[!windows] exec rclone check $DST $WORK/rsyncDst\ No newline at end of filediff --git a/fs/logger/testdata/script/TestRepoCompare.txtar b/fs/logger/testdata/script/TestRepoCompare.txtarnew file mode 100644index 000000000..4bbe9220a--- /dev/null+++ b/fs/logger/testdata/script/TestRepoCompare.txtar@@ -0,0 +1,52 @@+# tests whether rclone check and rclone sync output exactly the same file lists.+# uses two different old versions of rclone source code for the src and dst+# to produce a more realistic test than makefiles+# (has lots of files with same name but different content)++# Filling src and dst with two different versions of rclone source!:+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip+exec unzip $SRC/src.zip -d $SRC+exec unzip $DST/dst.zip -d $DST+++# running rclone check for baseline test:+# error is expected here:+! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q++# running sync with output files:+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt++# sorting them by line and diffing:+exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt+exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt+exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt+exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt+exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt+exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt++exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt++# diff match check vs. sync:+cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt+# diff combined check vs. sync:+cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt+# diff missingonsrc check vs. sync:+cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt+# diff missingondst check vs. sync:+cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt+# diff error check vs. sync:+cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt+# diff differ check vs. sync:+cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt++# verify accuracy+exec rclone check $SRC $DST+exec diff -rya --suppress-common-lines $SRC $DST+[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst+[!windows] exec rclone check $DST $WORK/rsyncDst\ No newline at end of filediff --git a/fs/operations/logger.go b/fs/operations/logger.goindex 9d0e213b4..e0447659d 100644--- a/fs/operations/logger.go+++ b/fs/operations/logger.go@@ -8,7 +8,6 @@ import ("io""github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/config/flags""github.com/rclone/rclone/fs/hash""github.com/spf13/pflag")@@ -107,28 +106,6 @@ type LoggerOpt struct {Absolute bool}-// AddLoggerFlags adds the logger flags to the cmdFlags command-func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *LoggerOpt, combined, missingOnSrc, missingOnDst, match, differ, errFile, destAfter *string) {- flags.StringVarP(cmdFlags, combined, "combined", "", *combined, "Make a combined report of changes to this file", "Sync")- flags.StringVarP(cmdFlags, missingOnSrc, "missing-on-src", "", *missingOnSrc, "Report all files missing from the source to this file", "Sync")- flags.StringVarP(cmdFlags, missingOnDst, "missing-on-dst", "", *missingOnDst, "Report all files missing from the destination to this file", "Sync")- flags.StringVarP(cmdFlags, match, "match", "", *match, "Report all matching files to this file", "Sync")- flags.StringVarP(cmdFlags, differ, "differ", "", *differ, "Report all non-matching files to this file", "Sync")- flags.StringVarP(cmdFlags, errFile, "error", "", *errFile, "Report all files with errors (hashing or reading) to this file", "Sync")- flags.StringVarP(cmdFlags, destAfter, "dest-after", "", *destAfter, "Report all files that exist on the dest post-sync", "Sync")-- // lsf flags for destAfter- flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")- flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")- flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")- flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")- flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")- flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")- flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")- flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")- // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")-}-// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = loggerfunc WithLogger(ctx context.Context, logger LoggerFn) context.Context {return context.WithValue(ctx, loggerKey, logger)diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.gonew file mode 100644index 000000000..80a6d77fb--- /dev/null+++ b/fs/operations/operationsflags/operationsflags.go@@ -0,0 +1,45 @@+// Package operationsflags defines the flags used by rclone operations.+// It is decoupled into a separate package so it can be replaced.+package operationsflags++import (+ "github.com/rclone/rclone/fs/config/flags"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/operations"+ "github.com/spf13/pflag"+)++// AddLoggerFlagsOptions contains options for the Logger Flags+type AddLoggerFlagsOptions struct {+ Combined string // a file with file names with leading sigils+ MissingOnSrc string // files only in the destination+ MissingOnDst string // files only in the source+ Match string // matching files+ Differ string // differing files+ ErrFile string // files with errors of some kind+ DestAfter string // files that exist on the destination post-sync+}++// AddLoggerFlags adds the logger flags to the cmdFlags command+func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {+ flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnSrc, "missing-on-src", "", flagsOpt.MissingOnSrc, "Report all files missing from the source to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnDst, "missing-on-dst", "", flagsOpt.MissingOnDst, "Report all files missing from the destination to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.Match, "match", "", flagsOpt.Match, "Report all matching files to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.Differ, "differ", "", flagsOpt.Differ, "Report all non-matching files to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.ErrFile, "error", "", flagsOpt.ErrFile, "Report all files with errors (hashing or reading) to this file", "Sync")+ flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")++ // lsf flags for destAfter+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")+ flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)", "")+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")+ opt.HashType = hash.MD5+ flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")+ // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")+}diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex e757af77c..4438477f4 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -1495,10 +1495,10 @@ func TestServerSideMoveOverlap(t *testing.T) {r.CheckRemoteItems(t, file1)// Subdir move with no filters should return ErrorCantMoveOverlapping- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)assert.EqualError(t, err, fs.ErrorOverlapping.Error())- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// Now try with a filter which should also fail with ErrorCantMoveOverlappingfi, err := filter.NewFilter(nil)@@ -1506,10 +1506,10 @@ func TestServerSideMoveOverlap(t *testing.T) {fi.Opt.MinSize = 40ctx = filter.ReplaceConfig(ctx, fi)- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)assert.EqualError(t, err, fs.ErrorOverlapping.Error())- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)}// Test a sync with overlap@@ -1822,9 +1822,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // not currently supportedrequire.NoError(t, err)file1dst := file1@@ -1838,9 +1838,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1b)accounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file1bdst := file1b@@ -1859,9 +1859,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c)accounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file2dst := file2@@ -1878,8 +1878,8 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5)accounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // ctx = predictDstFromLogger(ctx)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)err = Sync(ctx, fdst, r.Flocal, false)require.NoError(t, err)@@ -1890,9 +1890,9 @@ func TestSyncCopyDest(t *testing.T) {// check new dest, new copyaccounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)@@ -1904,9 +1904,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5, file7)accounting.GlobalStats().ResetCounters()- ctx = predictDstFromLogger(ctx)+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file7dst := file7@@ -2312,17 +2312,19 @@ func predictDstFromLogger(ctx context.Context) context.Context {file := winner.Objobj, ok := file.(fs.ObjectInfo)checksum := ""+ timeFormat := "2006-01-02 15:04:05"if ok {if obj.Fs().Hashes().GetOne() == hash.MD5 {// skip if no MD5- checksum, _ = obj.Hash(ctx, obj.Fs().Hashes().GetOne())+ checksum, _ = obj.Hash(ctx, hash.MD5)}+ timeFormat = operations.FormatForLSFPrecision(obj.Fs().Precision())}errMsg := ""if winner.Err != nil {errMsg = ";" + winner.Err.Error()}- operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), checksum, file.Size(), file.Remote(), errMsg)+ operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)}}return operations.WithSyncLogger(ctx, opt)@@ -2407,10 +2409,10 @@ func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, tnewlogger.Write(logger.Bytes())}- lsf := DstLsf(ctx, Fremote)- err := LoggerMatchesLsf(&newlogger, lsf)r := fstest.NewRun(t)if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash {+ lsf := DstLsf(ctx, Fremote)+ err := LoggerMatchesLsf(&newlogger, lsf)require.NoError(t, err)}}