Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

sync: report list of synced paths to file -- see #7282

nielash3a50f35df9ff
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index da9184f30..38076541e 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -3,22 +3,137 @@ package sync
import (
"context"
+ "io"
+ "os"
+
+ mutex "sync" // renamed as "sync" already in use
"github.com/rclone/rclone/cmd"
+ "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/operations/operationsflags"
"github.com/rclone/rclone/fs/sync"
"github.com/spf13/cobra"
)
var (
createEmptySrcDirs = false
+ opt = operations.LoggerOpt{}
+ loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
+ operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
+ // TODO: add same flags to move and copy
+}
+
+var lock mutex.Mutex
+
+func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
+ lock.Lock()
+ defer lock.Unlock()
+
+ if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ return
+ }
+
+ _, srcOk := src.(fs.Object)
+ _, dstOk := dst.(fs.Object)
+ var filename string
+ if !srcOk && !dstOk {
+ return
+ } else if srcOk && !dstOk {
+ filename = src.String()
+ } else {
+ filename = dst.String()
+ }
+
+ if sigil.Writer(opt) != nil {
+ operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
+ }
+ if opt.Combined != nil {
+ operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
+ fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
+ }
+ if opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ }
+}
+
+// GetSyncLoggerOpt gets the options corresponding to the logger flags
+func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
+ closers := []io.Closer{}
+
+ opt.LoggerFn = syncLoggerFn
+ if opt.TimeFormat == "max" {
+ opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
+ }
+ opt.SetListFormat(ctx, command.Flags())
+ opt.NewListJSON(ctx, fdst, "")
+
+ open := func(name string, pout *io.Writer) error {
+ if name == "" {
+ return nil
+ }
+ if name == "-" {
+ *pout = os.Stdout
+ return nil
+ }
+ out, err := os.Create(name)
+ if err != nil {
+ return err
+ }
+ *pout = out
+ closers = append(closers, out)
+ return nil
+ }
+
+ if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
+ return opt, nil, err
+ }
+ if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
+ return opt, nil, err
+ }
+
+ close := func() {
+ for _, closer := range closers {
+ err := closer.Close()
+ if err != nil {
+ fs.Errorf(nil, "Failed to close report output: %v", err)
+ }
+ }
+ }
+
+ return opt, close, nil
+}
+
+func anyNotBlank(s ...string) bool {
+ for _, x := range s {
+ if x != "" {
+ return true
+ }
+ }
+ return false
}
var commandDefinition = &cobra.Command{
@@ -59,6 +174,44 @@ destination that is inside the source directory.
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
+
+## Logger Flags
+
+The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
+ "`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
+stdout if it is ` + "`-`" + `) supplied. What they write is described in the
+help below. For example ` + "`--differ`" + ` will write all paths which are
+present on both the source and destination but different.
+
+The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- ` + "`= path`" + ` means path was found in source and destination and was identical
+- ` + "`- path`" + ` means path was missing on the source, so only in the destination
+- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
+- ` + "`* path`" + ` means path was present in source and destination but different.
+- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
+
+The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
+as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
+Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
+-- it should output an accurate list of what will be on the destination
+after the sync.
+
+Note that these logger flags have a few limitations, and certain scenarios
+are not currently supported:
+
+- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
+- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
+- server-side moves of an entire dir at once
+- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
+- Possibly some unusual error scenarios
+
+Note also that each file is logged during the sync, as opposed to after, so it
+is most useful as a predictor of what SHOULD happen to each file
+(which may or may not match what actually DID.)
`,
Annotations: map[string]string{
"groups": "Sync,Copy,Filter,Listing,Important",
@@ -67,10 +220,22 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14
cmd.CheckArgs(2, 2, command, args)
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
cmd.Run(true, true, command, func() error {
+ ctx := context.Background()
+ opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
+ if err != nil {
+ return err
+ }
+ defer close()
+
+ if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
+ loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
+ ctx = operations.WithSyncLogger(ctx, opt)
+ }
+
if srcFileName == "" {
- return sync.Sync(context.Background(), fdst, fsrc, createEmptySrcDirs)
+ return sync.Sync(ctx, fdst, fsrc, createEmptySrcDirs)
}
- return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
+ return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
})
},
}
diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.md
index 8096e01c3..5ecbb3c03 100644
--- a/docs/content/commands/rclone_sync.md
+++ b/docs/content/commands/rclone_sync.md
@@ -48,6 +48,43 @@ destination that is inside the source directory.
**Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
+## Logger Flags
+
+The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
+and `--error` flags write paths, one per line, to the file name (or
+stdout if it is `-`) supplied. What they write is described in the
+help below. For example `--differ` will write all paths which are
+present on both the source and destination but different.
+
+The `--combined` flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- `= path` means path was found in source and destination and was identical
+- `- path` means path was missing on the source, so only in the destination
+- `+ path` means path was missing on the destination, so only in the source
+- `* path` means path was present in source and destination but different.
+- `! path` means there was an error reading or hashing the source or dest.
+
+The `--dest-after` flag writes a list file using the same format flags
+as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
+Conceptually it is similar to rsync's `--itemize-changes`, but not identical
+-- it should output an accurate list of what will be on the destination
+after the sync.
+
+Note that these logger flags have a few limitations, and certain scenarios
+are not currently supported:
+
+- `--max-duration` / `CutoffModeHard`
+- `--compare-dest` / `--copy-dest`
+- server-side moves of an entire dir at once
+- High-level retries, because there would be duplicates (use `--retries 1` to disable)
+- Possibly some unusual error scenarios
+
+Note also that each file is logged during the sync, as opposed to after, so it
+is most useful as a predictor of what SHOULD happen to each file
+(which may or may not match what actually DID.)
```
rclone sync source:path dest:path [flags]
diff --git a/fs/logger/logger.go b/fs/logger/logger.go
new file mode 100644
index 000000000..722b5d396
--- /dev/null
+++ b/fs/logger/logger.go
@@ -0,0 +1,17 @@
+// Package logger implements testing for the sync (and bisync) logger
+package logger
+
+import (
+ _ "github.com/rclone/rclone/backend/all" // import all backends
+ "github.com/rclone/rclone/cmd"
+ _ "github.com/rclone/rclone/cmd/all" // import all commands
+ _ "github.com/rclone/rclone/lib/plugin" // import plugins
+)
+
+// Main enables the testscript package. See:
+// https://bitfieldconsulting.com/golang/cli-testing
+// https://pkg.go.dev/github.com/rogpeppe/[email protected]/testscript
+func Main() int {
+ cmd.Main()
+ return 0
+}
diff --git a/fs/logger/logger_test.go b/fs/logger/logger_test.go
new file mode 100644
index 000000000..0e53563ce
--- /dev/null
+++ b/fs/logger/logger_test.go
@@ -0,0 +1,33 @@
+package logger_test
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/rclone/rclone/fs/logger"
+ "github.com/rogpeppe/go-internal/testscript"
+)
+
+// TestMain drives the tests
+func TestMain(m *testing.M) {
+ // This enables the testscript package. See:
+ // https://bitfieldconsulting.com/golang/cli-testing
+ // https://pkg.go.dev/github.com/rogpeppe/[email protected]/testscript
+ os.Exit(testscript.RunMain(m, map[string]func() int{
+ "rclone": logger.Main,
+ }))
+}
+
+func TestLogger(t *testing.T) {
+ // Usage: https://bitfieldconsulting.com/golang/cli-testing
+
+ testscript.Run(t, testscript.Params{
+ Dir: "testdata/script",
+ Setup: func(env *testscript.Env) error {
+ env.Setenv("SRC", filepath.Join("$WORK", "src"))
+ env.Setenv("DST", filepath.Join("$WORK", "dst"))
+ return nil
+ },
+ })
+}
diff --git a/fs/logger/testdata/script/TestBeforeVsAfter.txtar b/fs/logger/testdata/script/TestBeforeVsAfter.txtar
new file mode 100644
index 000000000..0ccadc973
--- /dev/null
+++ b/fs/logger/testdata/script/TestBeforeVsAfter.txtar
@@ -0,0 +1,48 @@
+# tests whether an md5sum file generated post-sync matches our pre-sync prediction
+
+# Filling src and dst with two different versions of rclone source!:
+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
+exec unzip $SRC/src.zip -d $SRC
+exec unzip $DST/dst.zip -d $DST
+
+
+# generating sumfiles:
+exec rclone md5sum $SRC --output-file $WORK/src-before.txt
+exec rclone md5sum $DST --output-file $WORK/dst-before.txt
+
+# running sync with output files:
+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt --dest-after $WORK/SYNCdestafter.txt --format 'hp' --separator ' '
+
+# generating sumfiles:
+exec rclone md5sum $SRC --output-file $WORK/src-after.txt
+exec rclone md5sum $DST --output-file $WORK/dst-after.txt
+
+# sorting them by line and diffing:
+exec sort $WORK/src-before.txt -o $WORK/src-before.txt
+exec sort $WORK/dst-before.txt -o $WORK/dst-before.txt
+exec sort $WORK/src-after.txt -o $WORK/src-after.txt
+exec sort $WORK/dst-after.txt -o $WORK/dst-after.txt
+
+exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt
+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt
+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt
+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt
+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt
+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt
+exec sort $WORK/SYNCdestafter.txt -o $WORK/SYNCdestafter.txt
+
+# diff src before vs src after:
+cmp $WORK/src-before.txt $WORK/src-after.txt
+
+# diff dst before vs dst after:
+! cmp $WORK/dst-before.txt $WORK/dst-after.txt
+
+# diff src before vs dst after:
+cmp $WORK/src-before.txt $WORK/dst-after.txt
+
+# diff dst before vs src after:
+! cmp $WORK/dst-before.txt $WORK/src-after.txt
+
+# diff md5sum dst after vs sync dest-after:
+cmp $WORK/dst-after.txt $WORK/SYNCdestafter.txt
\ No newline at end of file
diff --git a/fs/logger/testdata/script/TestCheckVsSync.txtar b/fs/logger/testdata/script/TestCheckVsSync.txtar
new file mode 100644
index 000000000..11d2803d1
--- /dev/null
+++ b/fs/logger/testdata/script/TestCheckVsSync.txtar
@@ -0,0 +1,48 @@
+# tests whether rclone check and rclone sync output exactly the same file lists.
+
+# Filling src and dst with makefiles:
+exec rclone test makefiles $SRC --seed 0
+exec rclone test makefiles $DST --seed 1
+exec rclone test makefiles $SRC --seed 2
+exec rclone test makefiles $DST --seed 2
+
+# running rclone check for baseline test:
+# error is expected here:
+! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q
+
+# running sync with output files:
+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt
+
+# sorting them by line and diffing:
+exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt
+exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt
+exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt
+exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt
+exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt
+exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt
+
+exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt
+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt
+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt
+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt
+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt
+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt
+
+# diff match check vs. sync:
+cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt
+# diff combined check vs. sync:
+cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt
+# diff missingonsrc check vs. sync:
+cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt
+# diff missingondst check vs. sync:
+cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt
+# diff error check vs. sync:
+cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt
+# diff differ check vs. sync:
+cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt
+
+# verify accuracy
+exec rclone check $SRC $DST
+exec diff -rya --suppress-common-lines $SRC $DST
+[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst
+[!windows] exec rclone check $DST $WORK/rsyncDst
\ No newline at end of file
diff --git a/fs/logger/testdata/script/TestRepoCompare.txtar b/fs/logger/testdata/script/TestRepoCompare.txtar
new file mode 100644
index 000000000..4bbe9220a
--- /dev/null
+++ b/fs/logger/testdata/script/TestRepoCompare.txtar
@@ -0,0 +1,52 @@
+# tests whether rclone check and rclone sync output exactly the same file lists.
+# uses two different old versions of rclone source code for the src and dst
+# to produce a more realistic test than makefiles
+# (has lots of files with same name but different content)
+
+# Filling src and dst with two different versions of rclone source!:
+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
+exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
+exec unzip $SRC/src.zip -d $SRC
+exec unzip $DST/dst.zip -d $DST
+
+
+# running rclone check for baseline test:
+# error is expected here:
+! exec rclone check $SRC $DST --match $WORK/CHECKmatch.txt --combined $WORK/CHECKcombined.txt --missing-on-src $WORK/CHECKmissingonsrc.txt --missing-on-dst $WORK/CHECKmissingondst.txt --error $WORK/CHECKerr.txt --differ $WORK/CHECKdiffer.txt -q
+
+# running sync with output files:
+exec rclone sync $SRC $DST --match $WORK/SYNCmatch.txt --combined $WORK/SYNCcombined.txt --missing-on-src $WORK/SYNCmissingonsrc.txt --missing-on-dst $WORK/SYNCmissingondst.txt --error $WORK/SYNCerr.txt --differ $WORK/SYNCdiffer.txt
+
+# sorting them by line and diffing:
+exec sort $WORK/CHECKmatch.txt -o $WORK/CHECKmatch.txt
+exec sort $WORK/CHECKcombined.txt -o $WORK/CHECKcombined.txt
+exec sort $WORK/CHECKmissingonsrc.txt -o $WORK/CHECKmissingonsrc.txt
+exec sort $WORK/CHECKmissingondst.txt -o $WORK/CHECKmissingondst.txt
+exec sort $WORK/CHECKerr.txt -o $WORK/CHECKerr.txt
+exec sort $WORK/CHECKdiffer.txt -o $WORK/CHECKdiffer.txt
+
+exec sort $WORK/SYNCmatch.txt -o $WORK/SYNCmatch.txt
+exec sort $WORK/SYNCcombined.txt -o $WORK/SYNCcombined.txt
+exec sort $WORK/SYNCmissingonsrc.txt -o $WORK/SYNCmissingonsrc.txt
+exec sort $WORK/SYNCmissingondst.txt -o $WORK/SYNCmissingondst.txt
+exec sort $WORK/SYNCerr.txt -o $WORK/SYNCerr.txt
+exec sort $WORK/SYNCdiffer.txt -o $WORK/SYNCdiffer.txt
+
+# diff match check vs. sync:
+cmp $WORK/CHECKmatch.txt $WORK/SYNCmatch.txt
+# diff combined check vs. sync:
+cmp $WORK/CHECKcombined.txt $WORK/SYNCcombined.txt
+# diff missingonsrc check vs. sync:
+cmp $WORK/CHECKmissingonsrc.txt $WORK/SYNCmissingonsrc.txt
+# diff missingondst check vs. sync:
+cmp $WORK/CHECKmissingondst.txt $WORK/SYNCmissingondst.txt
+# diff error check vs. sync:
+cmp $WORK/CHECKerr.txt $WORK/SYNCerr.txt
+# diff differ check vs. sync:
+cmp $WORK/CHECKdiffer.txt $WORK/SYNCdiffer.txt
+
+# verify accuracy
+exec rclone check $SRC $DST
+exec diff -rya --suppress-common-lines $SRC $DST
+[!windows] exec rsync -aEvhPu $SRC/ $WORK/rsyncDst
+[!windows] exec rclone check $DST $WORK/rsyncDst
\ No newline at end of file
diff --git a/fs/operations/logger.go b/fs/operations/logger.go
index 9d0e213b4..e0447659d 100644
--- a/fs/operations/logger.go
+++ b/fs/operations/logger.go
@@ -8,7 +8,6 @@ import (
"io"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/spf13/pflag"
)
@@ -107,28 +106,6 @@ type LoggerOpt struct {
Absolute bool
}
-// AddLoggerFlags adds the logger flags to the cmdFlags command
-func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *LoggerOpt, combined, missingOnSrc, missingOnDst, match, differ, errFile, destAfter *string) {
- flags.StringVarP(cmdFlags, combined, "combined", "", *combined, "Make a combined report of changes to this file", "Sync")
- flags.StringVarP(cmdFlags, missingOnSrc, "missing-on-src", "", *missingOnSrc, "Report all files missing from the source to this file", "Sync")
- flags.StringVarP(cmdFlags, missingOnDst, "missing-on-dst", "", *missingOnDst, "Report all files missing from the destination to this file", "Sync")
- flags.StringVarP(cmdFlags, match, "match", "", *match, "Report all matching files to this file", "Sync")
- flags.StringVarP(cmdFlags, differ, "differ", "", *differ, "Report all non-matching files to this file", "Sync")
- flags.StringVarP(cmdFlags, errFile, "error", "", *errFile, "Report all files with errors (hashing or reading) to this file", "Sync")
- flags.StringVarP(cmdFlags, destAfter, "dest-after", "", *destAfter, "Report all files that exist on the dest post-sync", "Sync")
-
- // lsf flags for destAfter
- flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
- flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
- flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
- flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
- flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
- flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
- flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
- flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
- // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
-}
-
// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
return context.WithValue(ctx, loggerKey, logger)
diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go
new file mode 100644
index 000000000..80a6d77fb
--- /dev/null
+++ b/fs/operations/operationsflags/operationsflags.go
@@ -0,0 +1,45 @@
+// Package operationsflags defines the flags used by rclone operations.
+// It is decoupled into a separate package so it can be replaced.
+package operationsflags
+
+import (
+ "github.com/rclone/rclone/fs/config/flags"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/spf13/pflag"
+)
+
+// AddLoggerFlagsOptions contains options for the Logger Flags
+type AddLoggerFlagsOptions struct {
+ Combined string // a file with file names with leading sigils
+ MissingOnSrc string // files only in the destination
+ MissingOnDst string // files only in the source
+ Match string // matching files
+ Differ string // differing files
+ ErrFile string // files with errors of some kind
+ DestAfter string // files that exist on the destination post-sync
+}
+
+// AddLoggerFlags adds the logger flags to the cmdFlags command
+func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {
+ flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnSrc, "missing-on-src", "", flagsOpt.MissingOnSrc, "Report all files missing from the source to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnDst, "missing-on-dst", "", flagsOpt.MissingOnDst, "Report all files missing from the destination to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.Match, "match", "", flagsOpt.Match, "Report all matching files to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.Differ, "differ", "", flagsOpt.Differ, "Report all non-matching files to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.ErrFile, "error", "", flagsOpt.ErrFile, "Report all files with errors (hashing or reading) to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")
+
+ // lsf flags for destAfter
+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
+ flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)", "")
+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
+ opt.HashType = hash.MD5
+ flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
+ // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
+}
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index e757af77c..4438477f4 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -1495,10 +1495,10 @@ func TestServerSideMoveOverlap(t *testing.T) {
r.CheckRemoteItems(t, file1)
// Subdir move with no filters should return ErrorCantMoveOverlapping
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
fi, err := filter.NewFilter(nil)
@@ -1506,10 +1506,10 @@ func TestServerSideMoveOverlap(t *testing.T) {
fi.Opt.MinSize = 40
ctx = filter.ReplaceConfig(ctx, fi)
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
}
// Test a sync with overlap
@@ -1822,9 +1822,9 @@ func TestSyncCopyDest(t *testing.T) {
r.CheckLocalItems(t, file1)
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = Sync(ctx, fdst, r.Flocal, false)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported
require.NoError(t, err)
file1dst := file1
@@ -1838,9 +1838,9 @@ func TestSyncCopyDest(t *testing.T) {
r.CheckLocalItems(t, file1b)
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = Sync(ctx, fdst, r.Flocal, false)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
require.NoError(t, err)
file1bdst := file1b
@@ -1859,9 +1859,9 @@ func TestSyncCopyDest(t *testing.T) {
r.CheckLocalItems(t, file1c)
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = Sync(ctx, fdst, r.Flocal, false)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
require.NoError(t, err)
file2dst := file2
@@ -1878,8 +1878,8 @@ func TestSyncCopyDest(t *testing.T) {
r.CheckLocalItems(t, file1c, file5)
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // ctx = predictDstFromLogger(ctx)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
err = Sync(ctx, fdst, r.Flocal, false)
require.NoError(t, err)
@@ -1890,9 +1890,9 @@ func TestSyncCopyDest(t *testing.T) {
// check new dest, new copy
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = Sync(ctx, fdst, r.Flocal, false)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
require.NoError(t, err)
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
@@ -1904,9 +1904,9 @@ func TestSyncCopyDest(t *testing.T) {
r.CheckLocalItems(t, file1c, file5, file7)
accounting.GlobalStats().ResetCounters()
- ctx = predictDstFromLogger(ctx)
+ // ctx = predictDstFromLogger(ctx)
err = Sync(ctx, fdst, r.Flocal, false)
- testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
require.NoError(t, err)
file7dst := file7
@@ -2312,17 +2312,19 @@ func predictDstFromLogger(ctx context.Context) context.Context {
file := winner.Obj
obj, ok := file.(fs.ObjectInfo)
checksum := ""
+ timeFormat := "2006-01-02 15:04:05"
if ok {
if obj.Fs().Hashes().GetOne() == hash.MD5 {
// skip if no MD5
- checksum, _ = obj.Hash(ctx, obj.Fs().Hashes().GetOne())
+ checksum, _ = obj.Hash(ctx, hash.MD5)
}
+ timeFormat = operations.FormatForLSFPrecision(obj.Fs().Precision())
}
errMsg := ""
if winner.Err != nil {
errMsg = ";" + winner.Err.Error()
}
- operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), checksum, file.Size(), file.Remote(), errMsg)
+ operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)
}
}
return operations.WithSyncLogger(ctx, opt)
@@ -2407,10 +2409,10 @@ func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, t
newlogger.Write(logger.Bytes())
}
- lsf := DstLsf(ctx, Fremote)
- err := LoggerMatchesLsf(&newlogger, lsf)
r := fstest.NewRun(t)
if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash {
+ lsf := DstLsf(ctx, Fremote)
+ err := LoggerMatchesLsf(&newlogger, lsf)
require.NoError(t, err)
}
}