"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: add logger to log list of sync results -- fixes #7282
nielashc0968a0987e1
diff --git a/fs/operations/check.go b/fs/operations/check.goindex 239b3a72f..328be2128 100644--- a/fs/operations/check.go+++ b/fs/operations/check.go@@ -67,10 +67,10 @@ func (c *checkMarch) report(o fs.DirEntry, out io.Writer, sigil rune) {func (c *checkMarch) reportFilename(filename string, out io.Writer, sigil rune) {if out != nil {- syncFprintf(out, "%s\n", filename)+ SyncFprintf(out, "%s\n", filename)}if c.opt.Combined != nil {- syncFprintf(c.opt.Combined, "%c %s\n", sigil, filename)+ SyncFprintf(c.opt.Combined, "%c %s\n", sigil, filename)}}diff --git a/fs/operations/logger.go b/fs/operations/logger.gonew file mode 100644index 000000000..9d0e213b4--- /dev/null+++ b/fs/operations/logger.go@@ -0,0 +1,369 @@+package operations++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "io"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/flags"+ "github.com/rclone/rclone/fs/hash"+ "github.com/spf13/pflag"+)++// Sigil represents the rune (-+=*!?) used by Logger to categorize files by their match/differ/missing status.+type Sigil rune++// String converts sigil to more human-readable string+func (sigil Sigil) String() string {+ switch sigil {+ case '-':+ return "MissingOnSrc"+ case '+':+ return "MissingOnDst"+ case '=':+ return "Match"+ case '*':+ return "Differ"+ case '!':+ return "Error"+ // case '.':+ // return "Completed"+ case '?':+ return "Other"+ }+ return "unknown"+}++// Writer directs traffic from sigil -> LoggerOpt.Writer+func (sigil Sigil) Writer(opt LoggerOpt) io.Writer {+ switch sigil {+ case '-':+ return opt.MissingOnSrc+ case '+':+ return opt.MissingOnDst+ case '=':+ return opt.Match+ case '*':+ return opt.Differ+ case '!':+ return opt.Error+ }+ return nil+}++// Sigil constants+const (+ MissingOnSrc Sigil = '-'+ MissingOnDst Sigil = '+'+ Match Sigil = '='+ Differ Sigil = '*'+ TransferError Sigil = '!'+ Other Sigil = '?' // reserved but not currently used+)++// LoggerFn uses fs.DirEntry instead of fs.Object so it can include Dirs+// For LoggerFn example, see bisync.WriteResults() or sync.SyncLoggerFn()+// Usage example: s.logger(ctx, operations.Differ, src, dst, nil)+type LoggerFn func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error)+type loggerContextKey struct{}+type loggerOptContextKey struct{}++var loggerKey = loggerContextKey{}+var loggerOptKey = loggerOptContextKey{}++// LoggerOpt contains options for the Sync Logger functions+// TODO: refactor Check in here too?+type LoggerOpt struct {+ // Fdst, Fsrc fs.Fs // fses to check+ // Check checkFn // function to use for checking+ // OneWay bool // one way only?+ LoggerFn LoggerFn // function to use for logging+ Combined io.Writer // a file with file names with leading sigils+ MissingOnSrc io.Writer // files only in the destination+ MissingOnDst io.Writer // files only in the source+ Match io.Writer // matching files+ Differ io.Writer // differing files+ Error io.Writer // files with errors of some kind+ DestAfter io.Writer // files that exist on the destination post-sync+ JSON *bytes.Buffer // used by bisync to read/write struct as JSON+ DeleteModeOff bool //affects whether Logger expects MissingOnSrc to be deleted++ // lsf options for destAfter+ ListFormat ListFormat+ JSONOpt ListJSONOpt+ LJ *listJSON+ Format string+ TimeFormat string+ Separator string+ DirSlash bool+ // Recurse bool+ HashType hash.Type+ FilesOnly bool+ DirsOnly bool+ Csv bool+ Absolute bool+}++// AddLoggerFlags adds the logger flags to the cmdFlags command+func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *LoggerOpt, combined, missingOnSrc, missingOnDst, match, differ, errFile, destAfter *string) {+ flags.StringVarP(cmdFlags, combined, "combined", "", *combined, "Make a combined report of changes to this file", "Sync")+ flags.StringVarP(cmdFlags, missingOnSrc, "missing-on-src", "", *missingOnSrc, "Report all files missing from the source to this file", "Sync")+ flags.StringVarP(cmdFlags, missingOnDst, "missing-on-dst", "", *missingOnDst, "Report all files missing from the destination to this file", "Sync")+ flags.StringVarP(cmdFlags, match, "match", "", *match, "Report all matching files to this file", "Sync")+ flags.StringVarP(cmdFlags, differ, "differ", "", *differ, "Report all non-matching files to this file", "Sync")+ flags.StringVarP(cmdFlags, errFile, "error", "", *errFile, "Report all files with errors (hashing or reading) to this file", "Sync")+ flags.StringVarP(cmdFlags, destAfter, "dest-after", "", *destAfter, "Report all files that exist on the dest post-sync", "Sync")++ // lsf flags for destAfter+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")+ flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")+ // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")+}++// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger+func WithLogger(ctx context.Context, logger LoggerFn) context.Context {+ return context.WithValue(ctx, loggerKey, logger)+}++// WithLoggerOpt stores loggerOpt in ctx and returns a copy of ctx in which loggerOptKey = loggerOpt+func WithLoggerOpt(ctx context.Context, loggerOpt LoggerOpt) context.Context {+ return context.WithValue(ctx, loggerOptKey, loggerOpt)+}++// GetLogger attempts to retrieve LoggerFn from context, returns it if found, otherwise returns no-op function+func GetLogger(ctx context.Context) (LoggerFn, bool) {+ logger, ok := ctx.Value(loggerKey).(LoggerFn)+ if !ok {+ logger = func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {}+ }+ return logger, ok+}++// GetLoggerOpt attempts to retrieve LoggerOpt from context, returns it if found, otherwise returns NewLoggerOpt()+func GetLoggerOpt(ctx context.Context) LoggerOpt {+ loggerOpt, ok := ctx.Value(loggerOptKey).(LoggerOpt)+ if ok {+ return loggerOpt+ }+ return NewLoggerOpt()+}++// WithSyncLogger starts a new logger with the options passed in and saves it to ctx for retrieval later+func WithSyncLogger(ctx context.Context, opt LoggerOpt) context.Context {+ ctx = WithLoggerOpt(ctx, opt)+ return WithLogger(ctx, func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {+ if opt.LoggerFn != nil {+ opt.LoggerFn(ctx, sigil, src, dst, err)+ } else {+ SyncFprintf(opt.Combined, "%c %s\n", sigil, dst.Remote())+ }+ })+}++// NewLoggerOpt returns a new LoggerOpt struct with defaults+func NewLoggerOpt() LoggerOpt {+ opt := LoggerOpt{+ Combined: new(bytes.Buffer),+ MissingOnSrc: new(bytes.Buffer),+ MissingOnDst: new(bytes.Buffer),+ Match: new(bytes.Buffer),+ Differ: new(bytes.Buffer),+ Error: new(bytes.Buffer),+ DestAfter: new(bytes.Buffer),+ JSON: new(bytes.Buffer),+ }+ return opt+}++// Winner predicts which side (src or dst) should end up winning out on the dst.+type Winner struct {+ Obj fs.DirEntry // the object that should exist on dst post-sync, if any+ Side string // whether the winning object was from the src or dst+ Err error // whether there's an error preventing us from predicting winner correctly (not whether there was a sync error more generally)+}++// WinningSide can be called in a LoggerFn to predict what the dest will look like post-sync+//+// This attempts to account for every case in which dst (intentionally) does not match src after a sync.+//+// Known issues / cases we can't confidently predict yet:+//+// --max-duration / CutoffModeHard+// --compare-dest / --copy-dest (because equal() is called multiple times for the same file)+// server-side moves of an entire dir at once (because we never get the individual file objects in the dir)+// High-level retries, because there would be dupes (use --retries 1 to disable)+// Possibly some error scenarios+func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) Winner {+ winner := Winner{nil, "none", nil}+ opt := GetLoggerOpt(ctx)+ ci := fs.GetConfig(ctx)++ if err == fs.ErrorIsDir {+ winner.Err = err+ if sigil == MissingOnSrc {+ if (opt.DeleteModeOff || ci.DryRun) && dst != nil {+ winner.Obj = dst+ winner.Side = "dst" // whatever's on dst will remain so after DryRun+ return winner+ }+ return winner // none, because dst should just get deleted+ }+ if sigil == MissingOnDst && ci.DryRun {+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun+ } else if ci.DryRun && dst != nil {+ winner.Obj = dst+ winner.Side = "dst"+ } else if src != nil {+ winner.Obj = src+ winner.Side = "src"+ }+ return winner+ }++ _, srcOk := src.(fs.Object)+ _, dstOk := dst.(fs.Object)+ if !srcOk && !dstOk {+ return winner // none, because we don't have enough info to continue.+ }++ switch sigil {+ case MissingOnSrc:+ if opt.DeleteModeOff || ci.DryRun { // i.e. it's a copy, not sync (or it's a DryRun)+ winner.Obj = dst+ winner.Side = "dst" // whatever's on dst will remain so after DryRun+ return winner+ }+ return winner // none, because dst should just get deleted+ case Match, Differ, MissingOnDst:+ if sigil == MissingOnDst && ci.DryRun {+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun+ }+ winner.Obj = src+ winner.Side = "src" // presume dst will end up matching src unless changed below+ if sigil == Match && (ci.SizeOnly || ci.CheckSum || ci.IgnoreSize || ci.UpdateOlder || ci.NoUpdateModTime) {+ winner.Obj = dst+ winner.Side = "dst" // ignore any differences with src because of user flags+ }+ if ci.IgnoreTimes {+ winner.Obj = src+ winner.Side = "src" // copy src to dst unconditionally+ }+ if (sigil == Match || sigil == Differ) && (ci.IgnoreExisting || ci.Immutable) {+ winner.Obj = dst+ winner.Side = "dst" // dst should remain unchanged if it already exists (and we know it does because it's Match or Differ)+ }+ if ci.DryRun {+ winner.Obj = dst+ winner.Side = "dst" // dst should remain unchanged after DryRun (note that we handled MissingOnDst earlier)+ }+ return winner+ case TransferError:+ winner.Obj = dst+ winner.Side = "dst" // usually, dst should not change if there's an error+ if dst == nil {+ winner.Obj = src+ winner.Side = "src" // but if for some reason we have a src and not a dst, go with it+ }+ if winner.Obj != nil {+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errors.New("max transfer duration reached as set by --max-duration")) {+ winner.Err = err // we can't confidently predict what survives if CutoffModeHard+ }+ return winner // we know at least one of the objects+ }+ }+ // should only make it this far if it's TransferError and both src and dst are nil+ winner.Side = "none"+ winner.Err = fmt.Errorf("unknown case -- can't determine winner. %v", err)+ fs.Debugf(winner.Obj, "%v", winner.Err)+ return winner+}++// SetListFormat sets opt.ListFormat for destAfter+// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from+func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {+ // Work out if the separatorFlag was supplied or not+ separatorFlag := cmdFlags.Lookup("separator")+ separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed+ // Default the separator to , if using CSV+ if opt.Csv && !separatorFlagSupplied {+ opt.Separator = ","+ }++ var list ListFormat+ list.SetSeparator(opt.Separator)+ list.SetCSV(opt.Csv)+ list.SetDirSlash(opt.DirSlash)+ list.SetAbsolute(opt.Absolute)+ var JSONOpt = ListJSONOpt{+ NoModTime: true,+ NoMimeType: true,+ DirsOnly: opt.DirsOnly,+ FilesOnly: opt.FilesOnly,+ // Recurse: opt.Recurse,+ }++ for _, char := range opt.Format {+ switch char {+ case 'p':+ list.AddPath()+ case 't':+ list.AddModTime(opt.TimeFormat)+ JSONOpt.NoModTime = false+ case 's':+ list.AddSize()+ case 'h':+ list.AddHash(opt.HashType)+ JSONOpt.ShowHash = true+ JSONOpt.HashTypes = []string{opt.HashType.String()}+ case 'i':+ list.AddID()+ case 'm':+ list.AddMimeType()+ JSONOpt.NoMimeType = false+ case 'e':+ list.AddEncrypted()+ JSONOpt.ShowEncrypted = true+ case 'o':+ list.AddOrigID()+ JSONOpt.ShowOrigIDs = true+ case 'T':+ list.AddTier()+ case 'M':+ list.AddMetadata()+ JSONOpt.Metadata = true+ default:+ fs.Errorf(nil, "unknown format character %q", char)+ }+ }+ opt.ListFormat = list+ opt.JSONOpt = JSONOpt+}++// NewListJSON makes a new *listJSON for destAfter+func (opt *LoggerOpt) NewListJSON(ctx context.Context, fdst fs.Fs, remote string) {+ opt.LJ, _ = newListJSON(ctx, fdst, remote, &opt.JSONOpt)+ fs.Debugf(nil, "%v", opt.LJ)+}++// JSONEntry returns a *ListJSONItem for destAfter+func (opt *LoggerOpt) JSONEntry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) {+ return opt.LJ.entry(ctx, entry)+}++// PrintDestAfter writes a *ListJSONItem to opt.DestAfter+func (opt *LoggerOpt) PrintDestAfter(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {+ entry := WinningSide(ctx, sigil, src, dst, err)+ if entry.Obj != nil {+ JSONEntry, _ := opt.JSONEntry(ctx, entry.Obj)+ _, _ = fmt.Fprintln(opt.DestAfter, opt.ListFormat.Format(JSONEntry))+ }+}diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex d426c673c..efa3d4bab 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -180,12 +180,15 @@ func logModTimeUpload(dst fs.Object) {func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool {ci := fs.GetConfig(ctx)+ logger, _ := GetLogger(ctx)if sizeDiffers(ctx, src, dst) {fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())+ logger(ctx, Differ, src, dst, nil)return false}if opt.sizeOnly {fs.Debugf(src, "Sizes identical")+ logger(ctx, Match, src, dst, nil)return true}@@ -197,6 +200,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)same, ht, _ := CheckHashes(ctx, src, dst)if !same {fs.Debugf(src, "%v differ", ht)+ logger(ctx, Differ, src, dst, nil)return false}if ht == hash.None {@@ -210,6 +214,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)} else {fs.Debugf(src, "Size and %v of src and dst objects identical", ht)}+ logger(ctx, Match, src, dst, nil)return true}@@ -219,12 +224,14 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)modifyWindow := fs.GetModifyWindow(ctx, src.Fs(), dst.Fs())if modifyWindow == fs.ModTimeNotSupported {fs.Debugf(src, "Sizes identical")+ logger(ctx, Match, src, dst, nil)return true}dstModTime := dst.ModTime(ctx)dt := dstModTime.Sub(srcModTime)if dt < modifyWindow && dt > -modifyWindow {fs.Debugf(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow)+ logger(ctx, Match, src, dst, nil)return true}@@ -235,10 +242,12 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)same, ht, _ := CheckHashes(ctx, src, dst)if !same {fs.Debugf(src, "%v differ", ht)+ logger(ctx, Differ, src, dst, nil)return false}if ht == hash.None && !ci.RefreshTimes {// if couldn't check hash, return that they differ+ logger(ctx, Differ, src, dst, nil)return false}@@ -249,6 +258,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)// Error if objects are treated as immutableif ci.Immutable {fs.Errorf(dst, "Timestamp mismatch between immutable objects")+ logger(ctx, Differ, src, dst, nil)return false}// Update the mtime of the dst object here@@ -256,6 +266,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)if errors.Is(err, fs.ErrorCantSetModTime) {logModTimeUpload(dst)fs.Infof(dst, "src and dst identical but can't set mod time without re-uploading")+ logger(ctx, Differ, src, dst, nil)return false} else if errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete) {logModTimeUpload(dst)@@ -268,6 +279,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)fs.Errorf(dst, "failed to delete before re-upload: %v", err)}}+ logger(ctx, Differ, src, dst, nil)return false} else if err != nil {err = fs.CountError(err)@@ -277,6 +289,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)}}}+ logger(ctx, Match, src, dst, nil)return true}@@ -513,6 +526,8 @@ func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, berr := DeleteFileWithBackupDir(ctx, dst, backupDir)if err != nil {errorCount.Add(1)+ logger, _ := GetLogger(ctx)+ logger(ctx, TransferError, nil, dst, err)if fserrors.IsFatalError(err) {fs.Errorf(dst, "Got fatal error on delete: %s", err)fatalErrorCount.Add(1)@@ -670,12 +685,12 @@ var SyncPrintf = func(format string, a ...interface{}) {fmt.Printf(format, a...)}-// Synchronized fmt.Fprintf+// SyncFprintf - Synchronized fmt.Fprintf//// Ignores errors from Fprintf.//// Prints to stdout if w is nil-func syncFprintf(w io.Writer, format string, a ...interface{}) {+func SyncFprintf(w io.Writer, format string, a ...interface{}) {if w == nil || w == os.Stdout {SyncPrintf(format, a...)} else {@@ -747,7 +762,7 @@ func CountStringField(count int64, humanReadable bool, rawWidth int) string {func List(ctx context.Context, f fs.Fs, w io.Writer) error {ci := fs.GetConfig(ctx)return ListFn(ctx, f, func(o fs.Object) {- syncFprintf(w, "%s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), o.Remote())+ SyncFprintf(w, "%s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), o.Remote())})}@@ -764,7 +779,7 @@ func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {tr.Done(ctx, nil)}()modTime := o.ModTime(ctx)- syncFprintf(w, "%s %s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote())+ SyncFprintf(w, "%s %s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote())})}@@ -864,7 +879,7 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlfs.Errorf(o, "%v", fs.CountError(err))return}- syncFprintf(w, "%*s %s\n", width, sum, o.Remote())+ SyncFprintf(w, "%*s %s\n", width, sum, o.Remote())}()})wg.Wait()@@ -889,7 +904,7 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writereturn fmt.Errorf("hasher returned an error: %w", err)}width := hash.Width(ht, outputBase64)- syncFprintf(w, "%*s -\n", width, sum)+ SyncFprintf(w, "%*s -\n", width, sum)return nil}@@ -925,7 +940,7 @@ func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error {entries.ForDir(func(dir fs.Directory) {if dir != nil {- syncFprintf(w, "%s %13s %s %s\n", SizeStringField(dir.Size(), ci.HumanReadable, 12), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), CountStringField(dir.Items(), ci.HumanReadable, 9), dir.Remote())+ SyncFprintf(w, "%s %13s %s %s\n", SizeStringField(dir.Size(), ci.HumanReadable, 12), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), CountStringField(dir.Items(), ci.HumanReadable, 9), dir.Remote())}})return nil@@ -1494,18 +1509,22 @@ func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, Comp// transferred or not.func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {ci := fs.GetConfig(ctx)+ logger, _ := GetLogger(ctx)if dst == nil {fs.Debugf(src, "Need to transfer - File not found at Destination")+ logger(ctx, MissingOnDst, src, nil, nil)return true}// If we should ignore existing files, don't transferif ci.IgnoreExisting {fs.Debugf(src, "Destination exists, skipping")+ logger(ctx, Match, src, dst, nil)return false}// If we should upload unconditionallyif ci.IgnoreTimes {fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use")+ logger(ctx, Differ, src, dst, nil)return true}// If UpdateOlder is in effect, skip if dst is newer than src@@ -1524,6 +1543,7 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {switch {case dt >= modifyWindow:fs.Debugf(src, "Destination is newer than source, skipping")+ logger(ctx, Match, src, dst, nil)return falsecase dt <= -modifyWindow:// force --checksum on for the check and do update modtimes by default@@ -1707,10 +1727,16 @@ func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err err// moveOrCopyFile moves or copies a single file possibly to a new namefunc moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {ci := fs.GetConfig(ctx)+ logger, usingLogger := GetLogger(ctx)dstFilePath := path.Join(fdst.Root(), dstFileName)srcFilePath := path.Join(fsrc.Root(), srcFileName)if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName)+ if usingLogger {+ srcObj, _ := fsrc.NewObject(ctx, srcFileName)+ dstObj, _ := fsrc.NewObject(ctx, dstFileName)+ logger(ctx, Match, srcObj, dstObj, nil)+ }return nil}@@ -1723,6 +1749,7 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str// Find src objectsrcObj, err := fsrc.NewObject(ctx, srcFileName)if err != nil {+ logger(ctx, TransferError, srcObj, nil, err)return err}@@ -1733,6 +1760,7 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName strif errors.Is(err, fs.ErrorObjectNotFound) {dstObj = nil} else if err != nil {+ logger(ctx, TransferError, nil, dstObj, err)return err}}@@ -1744,11 +1772,13 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName strif !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath) {// Create random name to temporarily move file totmpObjName := dstFileName + "-rclone-move-" + random.String(8)- _, err := fdst.NewObject(ctx, tmpObjName)+ tmpObjFail, err := fdst.NewObject(ctx, tmpObjName)if err != fs.ErrorObjectNotFound {if err == nil {+ logger(ctx, TransferError, nil, tmpObjFail, err)return errors.New("found an already existing file with a randomly generated name. Try the operation again")}+ logger(ctx, TransferError, nil, tmpObjFail, err)return fmt.Errorf("error while attempting to move file to a temporary location: %w", err)}tr := accounting.Stats(ctx).NewTransfer(srcObj)@@ -1757,9 +1787,11 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str}()tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj)if err != nil {+ logger(ctx, TransferError, srcObj, tmpObj, err)return fmt.Errorf("error while moving file to temporary location: %w", err)}_, err = Op(ctx, fdst, nil, dstFileName, tmpObj)+ logger(ctx, MissingOnDst, tmpObj, nil, err)return err}@@ -1797,9 +1829,11 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName strif dstObj != nil && backupDir != nil {err = MoveBackupDir(ctx, backupDir, dstObj)if err != nil {+ logger(ctx, TransferError, dstObj, nil, err)return fmt.Errorf("moving to --backup-dir failed: %w", err)}// If successful zero out the dstObj as it is no longer there+ logger(ctx, MissingOnDst, dstObj, nil, nil)dstObj = nil}@@ -1808,8 +1842,10 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName strif !cp {if ci.IgnoreExisting {fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")+ logger(ctx, Match, srcObj, dstObj, nil)} else {err = DeleteFile(ctx, srcObj)+ logger(ctx, Differ, srcObj, dstObj, nil)}}}diff --git a/fs/sync/sync.go b/fs/sync/sync.goindex 18a8b67b7..9154171c5 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -82,6 +82,8 @@ type syncCopyMove struct {backupDir fs.Fs // place to store overwrites/deletescheckFirst bool // if set run all the checkers before starting transfersmaxDurationEndTime time.Time // end time if --max-duration is set+ logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer+ usingLogger bool // whether we are using logger}type trackRenamesStrategy byte@@ -135,6 +137,16 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeletetrackRenamesCh: make(chan fs.Object, ci.Checkers),checkFirst: ci.CheckFirst,}++ s.logger, s.usingLogger = operations.GetLogger(ctx)++ if deleteMode == fs.DeleteModeOff {+ loggerOpt := operations.GetLoggerOpt(ctx)+ loggerOpt.DeleteModeOff = true+ loggerOpt.LoggerFn = s.logger+ ctx = operations.WithLoggerOpt(ctx, loggerOpt)+ }+backlog := ci.MaxBacklogif s.checkFirst {fs.Infof(s.fdst, "Running all checks before starting transfers")@@ -345,6 +357,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.WNoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)if err != nil {s.processError(err)+ s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err)}if NoNeedTransfer {needTransfer = false@@ -362,6 +375,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.Werr := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst)if err != nil {s.processError(err)+ s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err)} else {// If successful zero out the dst as it is no longer there and copy the filepair.Dst = nil@@ -394,7 +408,9 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.Wreturn}} else {- s.processError(operations.DeleteFile(s.ctx, src))+ deleteFileErr := operations.DeleteFile(s.ctx, src)+ s.processError(deleteFileErr)+ s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, deleteFileErr)}}}@@ -446,6 +462,9 @@ func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs,_, err = operations.Copy(ctx, fdst, dst, src.Remote(), src)}s.processError(err)+ if err != nil {+ s.logger(ctx, operations.TransferError, src, dst, err)+ }}}@@ -556,6 +575,16 @@ func (s *syncCopyMove) stopDeleters() {func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {if accounting.Stats(s.ctx).Errored() && !s.ci.IgnoreErrors {fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)+ // log all deletes as errors+ for remote, o := range s.dstFiles {+ if checkSrcMap {+ _, exists := s.srcFiles[remote]+ if exists {+ continue+ }+ }+ s.logger(s.ctx, operations.TransferError, nil, o, fs.ErrorNotDeleting)+ }return fs.ErrorNotDeleting}@@ -967,10 +996,23 @@ func (s *syncCopyMove) run() error {// DstOnly have an object which is in the destination onlyfunc (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {if s.deleteMode == fs.DeleteModeOff {+ if s.usingLogger {+ switch x := dst.(type) {+ case fs.Object:+ s.logger(s.ctx, operations.MissingOnSrc, nil, x, nil)+ case fs.Directory:+ // it's a directory that we'd normally skip, because we're not deleting anything on the dest+ // however, to make sure every file is logged, we need to list it, so we need to return true here.+ // we skip this when not using logger.+ s.logger(s.ctx, operations.MissingOnSrc, nil, dst, fs.ErrorIsDir)+ return true+ }+ }return false}switch x := dst.(type) {case fs.Object:+ s.logger(s.ctx, operations.MissingOnSrc, nil, x, nil)switch s.deleteMode {case fs.DeleteModeAfter:// record object as needs deleting@@ -992,6 +1034,7 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {if s.fdst.Features().CanHaveEmptyDirectories {s.dstEmptyDirsMu.Lock()s.dstEmptyDirs[dst.Remote()] = dst+ s.logger(s.ctx, operations.MissingOnSrc, nil, dst, fs.ErrorIsDir)s.dstEmptyDirsMu.Unlock()}return true@@ -1009,6 +1052,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {}switch x := src.(type) {case fs.Object:+ s.logger(s.ctx, operations.MissingOnDst, x, nil, nil)// If it's a copy operation,// remove parent directory from srcEmptyDirs// since it's not really empty@@ -1028,6 +1072,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir)if err != nil {s.processError(err)+ s.logger(s.ctx, operations.TransferError, x, nil, err)}if !NoNeedTransfer {// No need to check since doesn't exist@@ -1044,6 +1089,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {s.srcEmptyDirsMu.Lock()s.srcParentDirCheck(src)s.srcEmptyDirs[src.Remote()] = src+ s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)s.srcEmptyDirsMu.Unlock()return truedefault:@@ -1065,6 +1111,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse}dstX, ok := dst.(fs.Object)if ok {+ // No logger here because we'll handle it in equal()ok = s.toBeChecked.Put(s.inCtx, fs.ObjectPair{Src: srcX, Dst: dstX})if !ok {return false@@ -1074,11 +1121,13 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurseerr := errors.New("can't overwrite directory with file")fs.Errorf(dst, "%v", err)s.processError(err)+ s.logger(ctx, operations.TransferError, srcX, dstX, err)}case fs.Directory:// Do the same thing to the entire contents of the directory_, ok := dst.(fs.Directory)if ok {+ s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)// Only record matched (src & dst) empty dirs when performing moveif s.DoMove {// Record the src directory for deletion@@ -1094,6 +1143,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurseerr := errors.New("can't overwrite file with directory")fs.Errorf(dst, "%v", err)s.processError(err)+ s.logger(ctx, operations.TransferError, src.(fs.ObjectInfo), dst.(fs.ObjectInfo), err)default:panic("Bad object in DirEntries")}diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex f24a783ed..e757af77c 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -3,14 +3,20 @@package syncimport (+ "bytes""context""errors""fmt"+ "os"+ "os/exec""runtime"+ "sort""strings""testing""time"+ mutex "sync" // renamed as "sync" already in use+_ "github.com/rclone/rclone/backend/all" // import all backends"github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting"@@ -45,7 +51,9 @@ func TestCopyWithDryRun(t *testing.T) {r.Mkdir(ctx, r.Fremote)ci.DryRun = true+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // error expected here because dry-runrequire.NoError(t, err)r.CheckLocalItems(t, file1)@@ -59,8 +67,10 @@ func TestCopy(t *testing.T) {file1 := r.WriteFile("sub dir/hello world", "hello world", t1)r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -76,8 +86,10 @@ func TestCopyMissingDirectory(t *testing.T) {t.Fatal(err)}+ ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, r.Fremote, nonExistingFs, false)require.Error(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)}// Now with --no-traverse@@ -90,8 +102,10 @@ func TestCopyNoTraverse(t *testing.T) {file1 := r.WriteFile("sub dir/hello world", "hello world", t1)+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -107,8 +121,10 @@ func TestCopyCheckFirst(t *testing.T) {file1 := r.WriteFile("sub dir/hello world", "hello world", t1)+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -125,8 +141,10 @@ func TestSyncNoTraverse(t *testing.T) {file1 := r.WriteFile("sub dir/hello world", "hello world", t1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -143,8 +161,10 @@ func TestCopyWithDepth(t *testing.T) {// Check the MaxDepth tooci.MaxDepth = 1+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t, file2)@@ -169,8 +189,10 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {ci.NoTraverse = noTraverse+ ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t, file1)@@ -187,8 +209,10 @@ func TestCopyEmptyDirectories(t *testing.T) {require.NoError(t, err)r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, r.Fremote, r.Flocal, true)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteListing(t,@@ -211,8 +235,10 @@ func TestMoveEmptyDirectories(t *testing.T) {require.NoError(t, err)r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, r.Fremote, r.Flocal, false, true)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteListing(t,@@ -235,8 +261,10 @@ func TestSyncEmptyDirectories(t *testing.T) {require.NoError(t, err)r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, true)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteListing(t,@@ -262,8 +290,10 @@ func TestServerSideCopy(t *testing.T) {defer finaliseCopy()t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)+ ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, FremoteCopy, r.Fremote, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)fstest.CheckItems(t, FremoteCopy, file1)}@@ -280,8 +310,10 @@ func TestCopyAfterDelete(t *testing.T) {err := operations.Mkdir(ctx, r.Flocal, "")require.NoError(t, err)+ ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t)r.CheckRemoteItems(t, file1)@@ -294,8 +326,10 @@ func TestCopyRedownload(t *testing.T) {file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)r.CheckRemoteItems(t, file1)+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Flocal, r.Fremote, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// Test with combined precision of local and remote as we copied it there and backr.CheckLocalListing(t, []fstest.Item{file1}, nil)@@ -314,8 +348,10 @@ func TestSyncBasedOnCheckSum(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred exactly one file.assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())@@ -326,8 +362,10 @@ func TestSyncBasedOnCheckSum(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred no filesassert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())@@ -348,8 +386,10 @@ func TestSyncSizeOnly(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred exactly one file.assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())@@ -360,8 +400,10 @@ func TestSyncSizeOnly(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred no filesassert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())@@ -382,8 +424,10 @@ func TestSyncIgnoreSize(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred exactly one file.assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())@@ -394,8 +438,10 @@ func TestSyncIgnoreSize(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred no filesassert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())@@ -411,8 +457,10 @@ func TestSyncIgnoreTimes(t *testing.T) {r.CheckRemoteItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred exactly 0 files because the// files were identical.@@ -421,8 +469,10 @@ func TestSyncIgnoreTimes(t *testing.T) {ci.IgnoreTimes = trueaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// We should have transferred exactly one file even though the// files were identical.@@ -441,18 +491,22 @@ func TestSyncIgnoreExisting(t *testing.T) {ci.IgnoreExisting = trueaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// Change everythingr.WriteFile("existing", "newpotatoes", t2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)// Items should not changer.CheckRemoteItems(t, file1)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)}func TestSyncIgnoreErrors(t *testing.T) {@@ -490,8 +544,10 @@ func TestSyncIgnoreErrors(t *testing.T) {)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)_ = fs.CountError(errors.New("boom"))assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,@@ -530,8 +586,10 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {ci.DryRun = trueaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file2)@@ -539,8 +597,10 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {ci.DryRun = falseaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -565,8 +625,10 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {r.CheckRemoteItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file2)@@ -586,8 +648,10 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {r.CheckRemoteItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -606,8 +670,10 @@ func TestSyncAfterAddingAFile(t *testing.T) {r.CheckRemoteItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t, file1, file2)}@@ -621,8 +687,10 @@ func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file2)r.CheckRemoteItems(t, file2)}@@ -644,8 +712,10 @@ func TestSyncAfterChangingContentsOnly(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file2)r.CheckRemoteItems(t, file2)}@@ -661,9 +731,11 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {ci.DryRun = trueaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)ci.DryRun = falserequire.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file3, file1)r.CheckRemoteItems(t, file3, file2)@@ -679,8 +751,10 @@ func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T)r.CheckLocalItems(t, file1, file3)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1, file3)r.CheckRemoteItems(t, file1, file3)}@@ -724,8 +798,10 @@ func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *test)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,@@ -788,10 +864,12 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {},)+ ctx = predictDstFromLogger(ctx)accounting.GlobalStats().ResetCounters()_ = fs.CountError(errors.New("boom"))err := Sync(ctx, r.Fremote, r.Flocal, false)assert.Equal(t, fs.ErrorNotDeleting, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,@@ -861,8 +939,10 @@ func TestCopyDeleteBefore(t *testing.T) {r.CheckLocalItems(t, file2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, file1, file2)r.CheckLocalItems(t, file2)@@ -884,15 +964,19 @@ func TestSyncWithExclude(t *testing.T) {ctx = filter.ReplaceConfig(ctx, fi)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, file2, file1)// Now sync the other way round and check enormous doesn't get// deleted as it is excluded from the syncaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Flocal, r.Fremote, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file2, file1, file3)}@@ -913,15 +997,19 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {ctx = filter.ReplaceConfig(ctx, fi)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, file2)// Check sync the other way round to make sure enormous gets// deleted even though it is excludedaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Flocal, r.Fremote, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file2)}@@ -950,9 +1038,11 @@ func TestSyncWithUpdateOlder(t *testing.T) {ci.UpdateOlder = trueci.ModifyWindow = fs.ModTimeNotSupported+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)r.CheckRemoteItems(t, oneO, twoF, threeO, fourF, fiveF)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // no modtimeif r.Fremote.Hashes().Count() == 0 {t.Logf("Skip test with --checksum as no hashes supported")@@ -994,8 +1084,10 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {r.CheckRemoteItems(t)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)startTime := time.Now()err := Sync(ctx, r.Fremote, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.True(t, errors.Is(err, ErrorMaxDurationReached))if cutoffMode == fs.CutoffModeHard {@@ -1042,7 +1134,9 @@ func TestSyncWithTrackRenames(t *testing.T) {f2 := r.WriteFile("yam", "Yam Content", t2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)r.CheckLocalItems(t, f1, f2)@@ -1051,7 +1145,9 @@ func TestSyncWithTrackRenames(t *testing.T) {f2 = r.RenameFile(f2, "yaml")accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)@@ -1110,7 +1206,9 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {f2 := r.WriteFile("yam", "Yam Content", t2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)r.CheckLocalItems(t, f1, f2)@@ -1119,7 +1217,9 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {f2 = r.RenameFile(f2, "yaml")accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)@@ -1145,7 +1245,9 @@ func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {f2 := r.WriteFile("sub/yam", "Yam Content", t2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)r.CheckLocalItems(t, f1, f2)@@ -1154,7 +1256,9 @@ func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {f2 = r.RenameFile(f2, "yam")accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckRemoteItems(t, f1, f2)@@ -1200,8 +1304,10 @@ func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFi// Do server-side moveaccounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx) // not currently supported -- doesn't list all contents of dir.err = MoveDir(ctx, FremoteMove, r.Fremote, testDeleteEmptyDirs, false)require.NoError(t, err)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)if withFilter {r.CheckRemoteItems(t, file2)@@ -1227,8 +1333,10 @@ func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFi// Move it back to a new empty remote, dst does not exist this timeaccounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)require.NoError(t, err)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)if withFilter {fstest.CheckItems(t, FremoteMove2, file1, file3u)@@ -1252,8 +1360,10 @@ func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {r.Mkdir(ctx, r.Fremote)// run move with --delete-empty-src-dirs+ ctx = predictDstFromLogger(ctx)err := MoveDir(ctx, r.Fremote, r.Flocal, true, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,@@ -1270,8 +1380,10 @@ func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {file2 := r.WriteFile("nested/sub dir/file", "nested", t1)r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,@@ -1295,8 +1407,10 @@ func TestMoveWithIgnoreExisting(t *testing.T) {ci.IgnoreExisting = trueaccounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)require.NoError(t, err)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalListing(t,[]fstest.Item{},@@ -1314,8 +1428,10 @@ func TestMoveWithIgnoreExisting(t *testing.T) {// Recreate first file with modified contentfile1b := r.WriteFile("existing", "newpotatoes", t2)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, r.Fremote, r.Flocal, false, false)require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// Source items should still exist in modified stater.CheckLocalListing(t,@@ -1379,8 +1495,10 @@ func TestServerSideMoveOverlap(t *testing.T) {r.CheckRemoteItems(t, file1)// Subdir move with no filters should return ErrorCantMoveOverlapping+ ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)assert.EqualError(t, err, fs.ErrorOverlapping.Error())+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)// Now try with a filter which should also fail with ErrorCantMoveOverlappingfi, err := filter.NewFilter(nil)@@ -1388,8 +1506,10 @@ func TestServerSideMoveOverlap(t *testing.T) {fi.Opt.MinSize = 40ctx = filter.ReplaceConfig(ctx, fi)+ ctx = predictDstFromLogger(ctx)err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)assert.EqualError(t, err, fs.ErrorOverlapping.Error())+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)}// Test a sync with overlap@@ -1407,10 +1527,18 @@ func TestSyncOverlap(t *testing.T) {assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())}+ ctx = predictDstFromLogger(ctx)checkErr(Sync(ctx, FremoteSync, r.Fremote, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ ctx = predictDstFromLogger(ctx)checkErr(Sync(ctx, r.Fremote, FremoteSync, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ ctx = predictDstFromLogger(ctx)checkErr(Sync(ctx, r.Fremote, r.Fremote, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ ctx = predictDstFromLogger(ctx)checkErr(Sync(ctx, FremoteSync, FremoteSync, false))+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)}// Test a sync with filtered overlap@@ -1453,29 +1581,63 @@ func TestSyncOverlapWithFilter(t *testing.T) {}accounting.GlobalStats().ResetCounters()+ filterCtx = predictDstFromLogger(filterCtx)checkNoErr(Sync(filterCtx, FremoteSync, r.Fremote, false))checkErr(Sync(ctx, FremoteSync, r.Fremote, false))checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, r.Fremote, FremoteSync, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(filterCtx, r.Fremote, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, r.Fremote, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(filterCtx, FremoteSync, FremoteSync, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, FremoteSync, FremoteSync, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkNoErr(Sync(filterCtx, FremoteSync2, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, FremoteSync2, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync2, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, r.Fremote, FremoteSync2, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(filterCtx, FremoteSync2, FremoteSync2, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, FremoteSync2, FremoteSync2, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkNoErr(Sync(filterCtx, FremoteSync3, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, FremoteSync3, r.Fremote, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)// Destination is excluded so this test makes no sense// checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false))checkErr(Sync(ctx, r.Fremote, FremoteSync3, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(filterCtx, FremoteSync3, FremoteSync3, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)+ filterCtx = predictDstFromLogger(filterCtx)checkErr(Sync(ctx, FremoteSync3, FremoteSync3, false))+ testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)}// Test with CompareDest set@@ -1494,7 +1656,9 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx) // not currently supported due to duplicate equal() checkserr = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file1dst := file1@@ -1508,7 +1672,9 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1b)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file1bdst := file1b@@ -1524,7 +1690,9 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1c)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file3)@@ -1536,14 +1704,18 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file3, file4)// check new dest, new compareaccounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file3, file4)@@ -1570,7 +1742,9 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5b)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file3, file4)@@ -1584,7 +1758,9 @@ func TestSyncCompareDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5c)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file5cdst := file5c@@ -1615,7 +1791,9 @@ func TestSyncMultipleCompareDest(t *testing.T) {accounting.GlobalStats().ResetCounters()fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")require.NoError(t, err)+ // ctx = predictDstFromLogger(ctx)require.NoError(t, Sync(ctx, fdst, r.Flocal, false))+ // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)fdest3 := fsrc3fdest3.Path = "dest/3"@@ -1644,7 +1822,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file1dst := file1@@ -1658,7 +1838,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1b)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file1bdst := file1b@@ -1677,7 +1859,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file2dst := file2@@ -1694,6 +1878,8 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)err = Sync(ctx, fdst, r.Flocal, false)require.NoError(t, err)@@ -1704,7 +1890,9 @@ func TestSyncCopyDest(t *testing.T) {// check new dest, new copyaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)@@ -1716,7 +1904,9 @@ func TestSyncCopyDest(t *testing.T) {r.CheckLocalItems(t, file1c, file5, file7)accounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, fdst, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)file7dst := file7@@ -1928,7 +2118,9 @@ func TestSyncUTFNorm(t *testing.T) {r.CheckRemoteItems(t, file2)accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOSrequire.NoError(t, err)// We should have transferred exactly one file, but kept the@@ -1954,7 +2146,9 @@ func TestSyncImmutable(t *testing.T) {// Should succeedaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)require.NoError(t, err)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)@@ -1966,7 +2160,9 @@ func TestSyncImmutable(t *testing.T) {// Should fail with ErrorImmutableModified and not modify local or remote filesaccounting.GlobalStats().ResetCounters()+ ctx = predictDstFromLogger(ctx)err = Sync(ctx, r.Fremote, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)assert.EqualError(t, err, fs.ErrorImmutableModified.Error())r.CheckLocalItems(t, file2)r.CheckRemoteItems(t, file1)@@ -1993,7 +2189,9 @@ func TestSyncIgnoreCase(t *testing.T) {// Should not copy files that are differently-cased but otherwise identicalaccounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOSrequire.NoError(t, err)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file2)@@ -2025,7 +2223,9 @@ func TestMaxTransfer(t *testing.T) {accounting.GlobalStats().ResetCounters()+ // ctx = predictDstFromLogger(ctx) // not currently supportederr := Sync(ctx, r.Fremote, r.Flocal, false)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)expectedErr := fserrors.FsError(accounting.ErrorMaxTransferLimitReachedFatal)if cutoff != fs.CutoffModeHard {expectedErr = accounting.ErrorMaxTransferLimitReachedGraceful@@ -2075,7 +2275,9 @@ func testSyncConcurrent(t *testing.T, subtest string) {r.CheckRemoteItems(t, itemsBefore...)stats.ResetErrors()+ ctx = predictDstFromLogger(ctx)err := Sync(ctx, r.Fremote, r.Flocal, false)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)if errors.Is(err, fs.ErrorCantUploadEmptyFiles) {t.Skipf("Skip test because remote cannot upload empty files")}@@ -2091,3 +2293,124 @@ func TestSyncConcurrentDelete(t *testing.T) {func TestSyncConcurrentTruncate(t *testing.T) {testSyncConcurrent(t, "truncate")}++// for testing logger:+func predictDstFromLogger(ctx context.Context) context.Context {+ opt := operations.NewLoggerOpt()+ var lock mutex.Mutex++ opt.LoggerFn = func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {+ lock.Lock()+ defer lock.Unlock()++ // ignore dirs for our purposes here+ if err == fs.ErrorIsDir {+ return+ }+ winner := operations.WinningSide(ctx, sigil, src, dst, err)+ if winner.Obj != nil {+ file := winner.Obj+ obj, ok := file.(fs.ObjectInfo)+ checksum := ""+ if ok {+ if obj.Fs().Hashes().GetOne() == hash.MD5 {+ // skip if no MD5+ checksum, _ = obj.Hash(ctx, obj.Fs().Hashes().GetOne())+ }+ }+ errMsg := ""+ if winner.Err != nil {+ errMsg = ";" + winner.Err.Error()+ }+ operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), checksum, file.Size(), file.Remote(), errMsg)+ }+ }+ return operations.WithSyncLogger(ctx, opt)+}++func DstLsf(ctx context.Context, Fremote fs.Fs) *bytes.Buffer {+ var opt = operations.ListJSONOpt{+ NoModTime: false,+ NoMimeType: true,+ DirsOnly: false,+ FilesOnly: true,+ Recurse: true,+ ShowHash: true,+ HashTypes: []string{"MD5"},+ }++ var list operations.ListFormat++ list.SetSeparator(";")+ timeFormat := operations.FormatForLSFPrecision(Fremote.Precision())+ list.AddModTime(timeFormat)+ list.AddHash(hash.MD5)+ list.AddSize()+ list.AddPath()++ out := new(bytes.Buffer)++ err := operations.ListJSON(ctx, Fremote, "", &opt, func(item *operations.ListJSONItem) error {+ _, _ = fmt.Fprintln(out, list.Format(item))+ return nil+ })+ if err != nil {+ fs.Errorf(Fremote, "ListJSON error: %v", err)+ }++ return out+}++func LoggerMatchesLsf(logger, lsf *bytes.Buffer) error {+ loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))+ sort.SliceStable(loggerSplit, func(i int, j int) bool { return string(loggerSplit[i]) < string(loggerSplit[j]) })+ lsfSplit := bytes.Split(lsf.Bytes(), []byte("\n"))+ sort.SliceStable(lsfSplit, func(i int, j int) bool { return string(lsfSplit[i]) < string(lsfSplit[j]) })++ loggerJoined := bytes.Join(loggerSplit, []byte("\n"))+ lsfJoined := bytes.Join(lsfSplit, []byte("\n"))++ if bytes.Equal(loggerJoined, lsfJoined) {+ return nil+ }+ Diff(string(loggerJoined), string(lsfJoined))+ return fmt.Errorf("logger does not match lsf! \nlogger: \n%s \nlsf: \n%s", loggerJoined, lsfJoined)+}++func Diff(rev1, rev2 string) {+ fmt.Printf("Diff of %q and %q\n", "logger", "lsf")+ cmd := exec.Command("bash", "-c", fmt.Sprintf(`diff <(echo "%s") <(echo "%s")`, rev1, rev2))+ out, _ := cmd.Output()+ _, _ = os.Stdout.Write(out)+}++func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, t *testing.T) {+ var newlogger bytes.Buffer+ canTestModtime := fs.GetModifyWindow(ctx, Fremote) != fs.ModTimeNotSupported+ canTestHash := Fremote.Hashes().Contains(hash.MD5)+ if !canTestHash || !canTestModtime {+ loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))+ for i, line := range loggerSplit {+ elements := bytes.Split(line, []byte(";"))+ if len(elements) >= 2 {+ if !canTestModtime {+ elements[0] = []byte("")+ }+ if !canTestHash {+ elements[1] = []byte("")+ }+ }+ loggerSplit[i] = bytes.Join(elements, []byte(";"))+ }+ newlogger.Write(bytes.Join(loggerSplit, []byte("\n")))+ } else {+ newlogger.Write(logger.Bytes())+ }++ lsf := DstLsf(ctx, Fremote)+ err := LoggerMatchesLsf(&newlogger, lsf)+ r := fstest.NewRun(t)+ if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash {+ require.NoError(t, err)+ }+}diff --git a/go.mod b/go.modindex 51f828948..8f1db25d0 100644--- a/go.mod+++ b/go.mod@@ -158,6 +158,7 @@ require (github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirectgithub.com/relvacode/iso8601 v1.3.0 // indirectgithub.com/rs/xid v1.5.0 // indirect+ github.com/rogpeppe/go-internal v1.11.0 // indirectgithub.com/russross/blackfriday/v2 v2.1.0 // indirectgithub.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirectgithub.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirectdiff --git a/go.sum b/go.sumindex 2e0dbbe73..ef1dd4fdc 100644--- a/go.sum+++ b/go.sum@@ -472,6 +472,7 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6pogithub.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=