"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: deglobalize to fix concurrent runs via rc - fixes #8675
nielash9073d173134f
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.goindex b4ed0fe93..57adbd988 100644--- a/cmd/bisync/bisync_test.go+++ b/cmd/bisync/bisync_test.go@@ -176,6 +176,7 @@ var (// Flag -refresh-times helps with Dropbox tests failing with message// "src and dst identical but can't set mod time without deleting and re-uploading"argRefreshTimes = flag.Bool("refresh-times", false, "Force refreshing the target modtime, useful for Dropbox (default: false)")+ ignoreLogs = flag.Bool("ignore-logs", false, "skip comparing log lines but still compare listings"))// bisyncTest keeps all test data in a single place@@ -264,6 +265,25 @@ func TestBisyncRemoteRemote(t *testing.T) {testBisync(t, remote, remote)}+// make sure rc can cope with running concurrent jobs+func TestBisyncConcurrent(t *testing.T) {+ oldArgTestCase := argTestCase+ *argTestCase = "basic"+ *ignoreLogs = true // not useful to compare logs here because both runs will be logging at once+ t.Cleanup(func() {+ argTestCase = oldArgTestCase+ *ignoreLogs = false+ })++ t.Run("test1", testParallel)+ t.Run("test2", testParallel)+}++func testParallel(t *testing.T) {+ t.Parallel()+ TestBisyncRemoteRemote(t)+}+// TestBisync is a test engine for bisync test cases.func testBisync(t *testing.T, path1, path2 string) {ctx := context.Background()@@ -1441,6 +1461,9 @@ func (b *bisyncTest) compareResults() int {resultText := b.mangleResult(b.workDir, file, false)if fileType(file) == "log" {+ if *ignoreLogs {+ continue+ }// save mangled logs so difference is easier on eyesgoldenFile := filepath.Join(b.logDir, "mangled.golden.log")resultFile := filepath.Join(b.logDir, "mangled.result.log")diff --git a/cmd/bisync/checkfn.go b/cmd/bisync/checkfn.goindex 9b0b9f144..f2015cb11 100644--- a/cmd/bisync/checkfn.go+++ b/cmd/bisync/checkfn.go@@ -16,15 +16,17 @@ import ("github.com/rclone/rclone/fs/operations")-var hashType hash.Type-var fsrc, fdst fs.Fs-var fcrypt *crypt.Fs+type bisyncCheck = struct {+ hashType hash.Type+ fsrc, fdst fs.Fs+ fcrypt *crypt.Fs+}// WhichCheck determines which CheckFn we should use based on the Fs types// It is more robust and accurate than Check because// it will fallback to CryptCheck or DownloadCheck instead of --size-only!// it returns the *operations.CheckOpt with the CheckFn set.-func WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.CheckOpt {+func (b *bisyncRun) WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.CheckOpt {ci := fs.GetConfig(ctx)common := opt.Fsrc.Hashes().Overlap(opt.Fdst.Hashes())@@ -40,32 +42,32 @@ func WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.Checkif (srcIsCrypt && dstIsCrypt) || (!srcIsCrypt && dstIsCrypt) {// if both are crypt or only dst is crypt- hashType = FdstCrypt.UnWrap().Hashes().GetOne()- if hashType != hash.None {+ b.check.hashType = FdstCrypt.UnWrap().Hashes().GetOne()+ if b.check.hashType != hash.None {// use cryptcheck- fsrc = opt.Fsrc- fdst = opt.Fdst- fcrypt = FdstCrypt- fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")- opt.Check = CryptCheckFn+ b.check.fsrc = opt.Fsrc+ b.check.fdst = opt.Fdst+ b.check.fcrypt = FdstCrypt+ fs.Infof(b.check.fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")+ opt.Check = b.CryptCheckFnreturn opt}} else if srcIsCrypt && !dstIsCrypt {// if only src is crypt- hashType = FsrcCrypt.UnWrap().Hashes().GetOne()- if hashType != hash.None {+ b.check.hashType = FsrcCrypt.UnWrap().Hashes().GetOne()+ if b.check.hashType != hash.None {// use reverse cryptcheck- fsrc = opt.Fdst- fdst = opt.Fsrc- fcrypt = FsrcCrypt- fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")- opt.Check = ReverseCryptCheckFn+ b.check.fsrc = opt.Fdst+ b.check.fdst = opt.Fsrc+ b.check.fcrypt = FsrcCrypt+ fs.Infof(b.check.fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")+ opt.Check = b.ReverseCryptCheckFnreturn opt}}// if we've gotten this far, neither check or cryptcheck will work, so use --download- fs.Infof(fdst, "Can't compare hashes, so using check --download for safety. (Use --size-only or --ignore-checksum to disable)")+ fs.Infof(b.check.fdst, "Can't compare hashes, so using check --download for safety. (Use --size-only or --ignore-checksum to disable)")opt.Check = DownloadCheckFnreturn opt}@@ -88,17 +90,17 @@ func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool,}// CryptCheckFn is a slightly modified version of CryptCheck-func CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {cryptDst := dst.(*crypt.Object)underlyingDst := cryptDst.UnWrap()- underlyingHash, err := underlyingDst.Hash(ctx, hashType)+ underlyingHash, err := underlyingDst.Hash(ctx, b.check.hashType)if err != nil {return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)}if underlyingHash == "" {return false, true, nil}- cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)+ cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)if err != nil {return true, false, fmt.Errorf("error computing hash: %w", err)}@@ -106,10 +108,10 @@ func CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHashreturn false, true, nil}if cryptHash != underlyingHash {- err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)+ err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)fs.Debugf(src, "%s", err.Error())// using same error msg as CheckFn so integration tests match- err = fmt.Errorf("%v differ", hashType)+ err = fmt.Errorf("%v differ", b.check.hashType)fs.Errorf(src, "%s", err.Error())return true, false, nil}@@ -118,8 +120,8 @@ func CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash// ReverseCryptCheckFn is like CryptCheckFn except src and dst are switched// result: src is crypt, dst is non-crypt-func ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {- return CryptCheckFn(ctx, src, dst)+func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+ return b.CryptCheckFn(ctx, src, dst)}// DownloadCheckFn is a slightly modified version of Check with --download@@ -137,7 +139,7 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filterif filterCheck.HaveFilesFrom() {fs.Debugf(nil, "There are potential conflicts to check.")- opt, close, checkopterr := check.GetCheckOpt(b.fs1, b.fs2)+ opt, close, checkopterr := check.GetCheckOpt(fs1, fs2)if checkopterr != nil {b.critical = trueb.retryable = true@@ -148,16 +150,16 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filteropt.Match = new(bytes.Buffer)- opt = WhichCheck(ctxCheck, opt)+ opt = b.WhichCheck(ctxCheck, opt)fs.Infof(nil, "Checking potential conflicts...")check := operations.CheckFn(ctxCheck, opt)fs.Infof(nil, "Finished checking the potential conflicts. %s", check)- //reset error count, because we don't want to count check errors as bisync errors+ // reset error count, because we don't want to count check errors as bisync errorsaccounting.Stats(ctxCheck).ResetErrors()- //return the list of identical files to check against later+ // return the list of identical files to check against laterif len(fmt.Sprint(opt.Match)) > 0 {matches = bilib.ToNames(strings.Split(fmt.Sprint(opt.Match), "\n"))}@@ -173,14 +175,14 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter// WhichEqual is similar to WhichCheck, but checks a single object.// Returns true if the objects are equal, false if they differ or if we don't know-func WhichEqual(ctx context.Context, src, dst fs.Object, Fsrc, Fdst fs.Fs) bool {+func (b *bisyncRun) WhichEqual(ctx context.Context, src, dst fs.Object, Fsrc, Fdst fs.Fs) bool {opt, close, checkopterr := check.GetCheckOpt(Fsrc, Fdst)if checkopterr != nil {fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)}defer close()- opt = WhichCheck(ctx, opt)+ opt = b.WhichCheck(ctx, opt)differ, noHash, err := opt.Check(ctx, dst, src)if err != nil {fs.Errorf(src, "failed to check: %v", err)@@ -217,7 +219,7 @@ func (b *bisyncRun) EqualFn(ctx context.Context) context.Context {equal, skipHash = timeSizeEqualFn()if equal && !skipHash {whichHashType := func(f fs.Info) hash.Type {- ht := getHashType(f.Name())+ ht := b.getHashType(f.Name())if ht == hash.None && b.opt.Compare.SlowHashSyncOnly && !b.opt.Resync {ht = f.Hashes().GetOne()}@@ -225,9 +227,9 @@ func (b *bisyncRun) EqualFn(ctx context.Context) context.Context {}srcHash, _ := src.Hash(ctx, whichHashType(src.Fs()))dstHash, _ := dst.Hash(ctx, whichHashType(dst.Fs()))- srcHash, _ = tryDownloadHash(ctx, src, srcHash)- dstHash, _ = tryDownloadHash(ctx, dst, dstHash)- equal = !hashDiffers(srcHash, dstHash, whichHashType(src.Fs()), whichHashType(dst.Fs()), src.Size(), dst.Size())+ srcHash, _ = b.tryDownloadHash(ctx, src, srcHash)+ dstHash, _ = b.tryDownloadHash(ctx, dst, dstHash)+ equal = !b.hashDiffers(srcHash, dstHash, whichHashType(src.Fs()), whichHashType(dst.Fs()), src.Size(), dst.Size())}if equal {logger(ctx, operations.Match, src, dst, nil)diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex 8ec2d393a..fd1f35e17 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -115,6 +115,7 @@ func (x *CheckSyncMode) Type() string {}// Opt keeps command line options+// internal functions should use b.opt insteadvar Opt Optionsfunc init() {diff --git a/cmd/bisync/compare.go b/cmd/bisync/compare.goindex 73ced57af..33079d5f7 100644--- a/cmd/bisync/compare.go+++ b/cmd/bisync/compare.go@@ -28,7 +28,7 @@ type CompareOpt = struct {DownloadHash bool}-func (b *bisyncRun) setCompareDefaults(ctx context.Context) error {+func (b *bisyncRun) setCompareDefaults(ctx context.Context) (err error) {ci := fs.GetConfig(ctx)// defaults@@ -120,25 +120,25 @@ func sizeDiffers(a, b int64) bool {// returns true if the hashes are definitely different.// returns false if equal, or if either is unknown.-func hashDiffers(a, b string, ht1, ht2 hash.Type, size1, size2 int64) bool {- if a == "" || b == "" {+func (b *bisyncRun) hashDiffers(stringA, stringB string, ht1, ht2 hash.Type, size1, size2 int64) bool {+ if stringA == "" || stringB == "" {if ht1 != hash.None && ht2 != hash.None && !(size1 <= 0 || size2 <= 0) {- fs.Logf(nil, Color(terminal.YellowFg, "WARNING: hash unexpectedly blank despite Fs support (%s, %s) (you may need to --resync!)"), a, b)+ fs.Logf(nil, Color(terminal.YellowFg, "WARNING: hash unexpectedly blank despite Fs support (%s, %s) (you may need to --resync!)"), stringA, stringB)}return false}if ht1 != ht2 {- if !(downloadHash && ((ht1 == hash.MD5 && ht2 == hash.None) || (ht1 == hash.None && ht2 == hash.MD5))) {+ if !(b.downloadHashOpt.downloadHash && ((ht1 == hash.MD5 && ht2 == hash.None) || (ht1 == hash.None && ht2 == hash.MD5))) {fs.Infof(nil, Color(terminal.YellowFg, "WARNING: Can't compare hashes of different types (%s, %s)"), ht1.String(), ht2.String())return false}}- return a != b+ return stringA != stringB}// chooses hash type, giving priority to types both sides have in commonfunc (b *bisyncRun) setHashType(ci *fs.ConfigInfo) {- downloadHash = b.opt.Compare.DownloadHash+ b.downloadHashOpt.downloadHash = b.opt.Compare.DownloadHashif b.opt.Compare.NoSlowHash && b.opt.Compare.SlowHashDetected {fs.Infof(nil, "Not checking for common hash as at least one slow hash detected.")} else {@@ -268,13 +268,15 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {return nil}-// downloadHash is true if we should attempt to compute hash by downloading when otherwise unavailable-var downloadHash bool-var downloadHashWarn mutex.Once-var firstDownloadHash mutex.Once+// b.downloadHashOpt.downloadHash is true if we should attempt to compute hash by downloading when otherwise unavailable+type downloadHashOpt struct {+ downloadHash bool+ downloadHashWarn mutex.Once+ firstDownloadHash mutex.Once+}-func tryDownloadHash(ctx context.Context, o fs.DirEntry, hashVal string) (string, error) {- if hashVal != "" || !downloadHash {+func (b *bisyncRun) tryDownloadHash(ctx context.Context, o fs.DirEntry, hashVal string) (string, error) {+ if hashVal != "" || !b.downloadHashOpt.downloadHash {return hashVal, nil}obj, ok := o.(fs.Object)@@ -283,14 +285,14 @@ func tryDownloadHash(ctx context.Context, o fs.DirEntry, hashVal string) (stringreturn hashVal, fs.ErrorObjectNotFound}if o.Size() < 0 {- downloadHashWarn.Do(func() {+ b.downloadHashOpt.downloadHashWarn.Do(func() {fs.Log(o, Color(terminal.YellowFg, "Skipping hash download as checksum not reliable with files of unknown length."))})fs.Debugf(o, "Skipping hash download as checksum not reliable with files of unknown length.")return hashVal, hash.ErrUnsupported}- firstDownloadHash.Do(func() {+ b.downloadHashOpt.firstDownloadHash.Do(func() {fs.Infoc(obj.Fs().Name(), Color(terminal.Dim, "Downloading hashes..."))})tr := accounting.Stats(ctx).NewCheckingTransfer(o, "computing hash with --download-hash")diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex 2fba379c6..b882fb564 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -219,7 +219,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,}}if b.opt.Compare.Checksum {- if hashDiffers(old.getHash(file), now.getHash(file), old.hash, now.hash, old.getSize(file), now.getSize(file)) {+ if b.hashDiffers(old.getHash(file), now.getHash(file), old.hash, now.hash, old.getSize(file), now.getSize(file)) {fs.Debugf(file, "(old: %v current: %v)", old.getHash(file), now.getHash(file))whatchanged = append(whatchanged, Color(terminal.MagentaFg, "hash"))d |= deltaHash@@ -346,7 +346,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (resultif d2.is(deltaOther) {// if size or hash differ, skip this, as we already know they're not equalif (b.opt.Compare.Size && sizeDiffers(ds1.size[file], ds2.size[file2])) ||- (b.opt.Compare.Checksum && hashDiffers(ds1.hash[file], ds2.hash[file2], b.opt.Compare.HashType1, b.opt.Compare.HashType2, ds1.size[file], ds2.size[file2])) {+ (b.opt.Compare.Checksum && b.hashDiffers(ds1.hash[file], ds2.hash[file2], b.opt.Compare.HashType1, b.opt.Compare.HashType2, ds1.size[file], ds2.size[file2])) {fs.Debugf(file, "skipping equality check as size/hash definitely differ")} else {checkit := func(filename string) {@@ -393,10 +393,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result// if files are identical, leave them alone instead of renamingif (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)- ls1.getPut(file, skippedDirs1)- ls2.getPut(file, skippedDirs2)+ b.march.ls1.getPut(file, skippedDirs1)+ b.march.ls2.getPut(file, skippedDirs2)b.debugFn(file, func() {- b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, ls2 has name?: %v", file, ls1.has(b.DebugName), ls2.has(b.DebugName)))+ b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, b.march.ls2 has name?: %v", file, b.march.ls1.has(b.DebugName), b.march.ls2.has(b.DebugName)))})} else {equal := matches.Has(file)@@ -409,16 +409,16 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result// the Path1 version is deemed "correct" in this scenariofs.Infof(alias, "Files are equal but will copy anyway to fix case to %s", file)copy1to2.Add(file)- } else if b.opt.Compare.Modtime && timeDiffers(ctx, ls1.getTime(ls1.getTryAlias(file, alias)), ls2.getTime(ls2.getTryAlias(file, alias)), b.fs1, b.fs2) {+ } else if b.opt.Compare.Modtime && timeDiffers(ctx, b.march.ls1.getTime(b.march.ls1.getTryAlias(file, alias)), b.march.ls2.getTime(b.march.ls2.getTryAlias(file, alias)), b.fs1, b.fs2) {fs.Infof(file, "Files are equal but will copy anyway to update modtime (will not rename)")- if ls1.getTime(ls1.getTryAlias(file, alias)).Before(ls2.getTime(ls2.getTryAlias(file, alias))) {+ if b.march.ls1.getTime(b.march.ls1.getTryAlias(file, alias)).Before(b.march.ls2.getTime(b.march.ls2.getTryAlias(file, alias))) {// Path2 is newerb.indent("Path2", p1, "Queue copy to Path1")- copy2to1.Add(ls2.getTryAlias(file, alias))+ copy2to1.Add(b.march.ls2.getTryAlias(file, alias))} else {// Path1 is newerb.indent("Path1", p2, "Queue copy to Path2")- copy1to2.Add(ls1.getTryAlias(file, alias))+ copy1to2.Add(b.march.ls1.getTryAlias(file, alias))}} else {fs.Infof(nil, "Files are equal! Skipping: %s", file)@@ -590,10 +590,10 @@ func (b *bisyncRun) updateAliases(ctx context.Context, ds1, ds2 *deltaSet) {fullMap1 := map[string]string{} // [transformedname]originalnamefullMap2 := map[string]string{} // [transformedname]originalname- for _, name := range ls1.list {+ for _, name := range b.march.ls1.list {fullMap1[transform(name)] = name}- for _, name := range ls2.list {+ for _, name := range b.march.ls2.list {fullMap2[transform(name)] = name}diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.goindex 1aa1e9a62..02f332a77 100644--- a/cmd/bisync/listing.go+++ b/cmd/bisync/listing.go@@ -202,8 +202,8 @@ func (b *bisyncRun) fileInfoEqual(file1, file2 string, ls1, ls2 *fileList) boolequal = false}}- if b.opt.Compare.Checksum && !ignoreListingChecksum {- if hashDiffers(ls1.getHash(file1), ls2.getHash(file2), b.opt.Compare.HashType1, b.opt.Compare.HashType2, ls1.getSize(file1), ls2.getSize(file2)) {+ if b.opt.Compare.Checksum && !b.queueOpt.ignoreListingChecksum {+ if b.hashDiffers(ls1.getHash(file1), ls2.getHash(file2), b.opt.Compare.HashType1, b.opt.Compare.HashType2, ls1.getSize(file1), ls2.getSize(file2)) {b.indent("ERROR", file1, fmt.Sprintf("Checksum not equal in listing. Path1: %v, Path2: %v", ls1.getHash(file1), ls2.getHash(file2)))equal = false}@@ -745,7 +745,7 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,if hashType != hash.None {hashVal, _ = obj.Hash(ctxRecheck, hashType)}- hashVal, _ = tryDownloadHash(ctxRecheck, obj, hashVal)+ hashVal, _ = b.tryDownloadHash(ctxRecheck, obj, hashVal)}var modtime time.Timeif b.opt.Compare.Modtime {@@ -759,7 +759,7 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,for _, dstObj := range dstObjs {if srcObj.Remote() == dstObj.Remote() || srcObj.Remote() == b.aliases.Alias(dstObj.Remote()) {// note: unlike Equal(), WhichEqual() does not update the modtime in dest if sums match but modtimes don't.- if b.opt.DryRun || WhichEqual(ctxRecheck, srcObj, dstObj, src, dst) {+ if b.opt.DryRun || b.WhichEqual(ctxRecheck, srcObj, dstObj, src, dst) {putObj(srcObj, srcList)putObj(dstObj, dstList)resolved = append(resolved, srcObj.Remote())@@ -773,7 +773,7 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,// skip and error during --resync, as rollback is not possibleif !slices.Contains(resolved, srcObj.Remote()) && !b.opt.DryRun {if b.opt.Resync {- err = errors.New("no dstObj match or files not equal")+ err := errors.New("no dstObj match or files not equal")b.handleErr(srcObj, "Unable to rollback during --resync", err, true, false)} else {toRollback = append(toRollback, srcObj.Remote())diff --git a/cmd/bisync/lockfile.go b/cmd/bisync/lockfile.goindex 90e8c2166..e294ff0e8 100644--- a/cmd/bisync/lockfile.go+++ b/cmd/bisync/lockfile.go@@ -16,16 +16,17 @@ import (const basicallyforever = fs.Duration(200 * 365 * 24 * time.Hour)-var stopRenewal func()--var data = struct {- Session string- PID string- TimeRenewed time.Time- TimeExpires time.Time-}{}+type lockFileOpt struct {+ stopRenewal func()+ data struct {+ Session string+ PID string+ TimeRenewed time.Time+ TimeExpires time.Time+ }+}-func (b *bisyncRun) setLockFile() error {+func (b *bisyncRun) setLockFile() (err error) {b.lockFile = ""b.setLockFileExpiration()if !b.opt.DryRun {@@ -45,24 +46,23 @@ func (b *bisyncRun) setLockFile() error {}fs.Debugf(nil, "Lock file created: %s", b.lockFile)b.renewLockFile()- stopRenewal = b.startLockRenewal()+ b.lockFileOpt.stopRenewal = b.startLockRenewal()}return nil}-func (b *bisyncRun) removeLockFile() {+func (b *bisyncRun) removeLockFile() (err error) {if b.lockFile != "" {- stopRenewal()- errUnlock := os.Remove(b.lockFile)- if errUnlock == nil {+ b.lockFileOpt.stopRenewal()+ err = os.Remove(b.lockFile)+ if err == nil {fs.Debugf(nil, "Lock file removed: %s", b.lockFile)- } else if err == nil {- err = errUnlock} else {- fs.Errorf(nil, "cannot remove lockfile %s: %v", b.lockFile, errUnlock)+ fs.Errorf(nil, "cannot remove lockfile %s: %v", b.lockFile, err)}b.lockFile = "" // block removing it again}+ return err}func (b *bisyncRun) setLockFileExpiration() {@@ -77,18 +77,18 @@ func (b *bisyncRun) setLockFileExpiration() {func (b *bisyncRun) renewLockFile() {if b.lockFile != "" && bilib.FileExists(b.lockFile) {- data.Session = b.basePath- data.PID = strconv.Itoa(os.Getpid())- data.TimeRenewed = time.Now()- data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))+ b.lockFileOpt.data.Session = b.basePath+ b.lockFileOpt.data.PID = strconv.Itoa(os.Getpid())+ b.lockFileOpt.data.TimeRenewed = time.Now()+ b.lockFileOpt.data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))// save data filedf, err := os.Create(b.lockFile)b.handleErr(b.lockFile, "error renewing lock file", err, true, true)- b.handleErr(b.lockFile, "error encoding JSON to lock file", json.NewEncoder(df).Encode(data), true, true)+ b.handleErr(b.lockFile, "error encoding JSON to lock file", json.NewEncoder(df).Encode(b.lockFileOpt.data), true, true)b.handleErr(b.lockFile, "error closing lock file", df.Close(), true, true)if b.opt.MaxLock < basicallyforever {- fs.Infof(nil, Color(terminal.HiBlueFg, "lock file renewed for %v. New expiration: %v"), b.opt.MaxLock, data.TimeExpires)+ fs.Infof(nil, Color(terminal.HiBlueFg, "lock file renewed for %v. New expiration: %v"), b.opt.MaxLock, b.lockFileOpt.data.TimeExpires)}}}@@ -99,7 +99,7 @@ func (b *bisyncRun) lockFileIsExpired() bool {b.handleErr(b.lockFile, "error reading lock file", err, true, true)dec := json.NewDecoder(rdf)for {- if err := dec.Decode(&data); err != nil {+ if err := dec.Decode(&b.lockFileOpt.data); err != nil {if err != io.EOF {fs.Errorf(b.lockFile, "err: %v", err)}@@ -107,14 +107,14 @@ func (b *bisyncRun) lockFileIsExpired() bool {}}b.handleErr(b.lockFile, "error closing file", rdf.Close(), true, true)- if !data.TimeExpires.IsZero() && data.TimeExpires.Before(time.Now()) {- fs.Infof(b.lockFile, Color(terminal.GreenFg, "Lock file found, but it expired at %v. Will delete it and proceed."), data.TimeExpires)+ if !b.lockFileOpt.data.TimeExpires.IsZero() && b.lockFileOpt.data.TimeExpires.Before(time.Now()) {+ fs.Infof(b.lockFile, Color(terminal.GreenFg, "Lock file found, but it expired at %v. Will delete it and proceed."), b.lockFileOpt.data.TimeExpires)markFailed(b.listing1) // listing is untrusted so force revert to prior (if --recover) or create new ones (if --resync)markFailed(b.listing2)return true}- fs.Infof(b.lockFile, Color(terminal.RedFg, "Valid lock file found. Expires at %v. (%v from now)"), data.TimeExpires, time.Since(data.TimeExpires).Abs().Round(time.Second))- prettyprint(data, "Lockfile info", fs.LogLevelInfo)+ fs.Infof(b.lockFile, Color(terminal.RedFg, "Valid lock file found. Expires at %v. (%v from now)"), b.lockFileOpt.data.TimeExpires, time.Since(b.lockFileOpt.data.TimeExpires).Abs().Round(time.Second))+ prettyprint(b.lockFileOpt.data, "Lockfile info", fs.LogLevelInfo)}return false}diff --git a/cmd/bisync/march.go b/cmd/bisync/march.goindex 60263e2d5..d2a3ecae9 100644--- a/cmd/bisync/march.go+++ b/cmd/bisync/march.go@@ -12,18 +12,20 @@ import ("github.com/rclone/rclone/fs/march")-var ls1 = newFileList()-var ls2 = newFileList()-var err error-var firstErr error-var marchAliasLock sync.Mutex-var marchLsLock sync.Mutex-var marchErrLock sync.Mutex-var marchCtx context.Context+type bisyncMarch struct {+ ls1 *fileList+ ls2 *fileList+ err error+ firstErr error+ marchAliasLock sync.Mutex+ marchLsLock sync.Mutex+ marchErrLock sync.Mutex+ marchCtx context.Context+}func (b *bisyncRun) makeMarchListing(ctx context.Context) (*fileList, *fileList, error) {ci := fs.GetConfig(ctx)- marchCtx = ctx+ b.march.marchCtx = ctxb.setupListing()fs.Debugf(b, "starting to march!")@@ -39,31 +41,31 @@ func (b *bisyncRun) makeMarchListing(ctx context.Context) (*fileList, *fileList,NoCheckDest: false,NoUnicodeNormalization: ci.NoUnicodeNormalization,}- err = m.Run(ctx)+ b.march.err = m.Run(ctx)- fs.Debugf(b, "march completed. err: %v", err)- if err == nil {- err = firstErr+ fs.Debugf(b, "march completed. err: %v", b.march.err)+ if b.march.err == nil {+ b.march.err = b.march.firstErr}- if err != nil {- b.handleErr("march", "error during march", err, true, true)+ if b.march.err != nil {+ b.handleErr("march", "error during march", b.march.err, true, true)b.abort = true- return ls1, ls2, err+ return b.march.ls1, b.march.ls2, b.march.err}// save files- if b.opt.Compare.DownloadHash && ls1.hash == hash.None {- ls1.hash = hash.MD5+ if b.opt.Compare.DownloadHash && b.march.ls1.hash == hash.None {+ b.march.ls1.hash = hash.MD5}- if b.opt.Compare.DownloadHash && ls2.hash == hash.None {- ls2.hash = hash.MD5+ if b.opt.Compare.DownloadHash && b.march.ls2.hash == hash.None {+ b.march.ls2.hash = hash.MD5}- err = ls1.save(ctx, b.newListing1)- b.handleErr(ls1, "error saving ls1 from march", err, true, true)- err = ls2.save(ctx, b.newListing2)- b.handleErr(ls2, "error saving ls2 from march", err, true, true)+ b.march.err = b.march.ls1.save(ctx, b.newListing1)+ b.handleErr(b.march.ls1, "error saving b.march.ls1 from march", b.march.err, true, true)+ b.march.err = b.march.ls2.save(ctx, b.newListing2)+ b.handleErr(b.march.ls2, "error saving b.march.ls2 from march", b.march.err, true, true)- return ls1, ls2, err+ return b.march.ls1, b.march.ls2, b.march.err}// SrcOnly have an object which is on path1 only@@ -83,9 +85,9 @@ func (b *bisyncRun) DstOnly(o fs.DirEntry) (recurse bool) {// Match is called when object exists on both path1 and path2 (whether equal or not)func (b *bisyncRun) Match(ctx context.Context, o2, o1 fs.DirEntry) (recurse bool) {fs.Debugf(o1, "both path1 and path2")- marchAliasLock.Lock()+ b.march.marchAliasLock.Lock()b.aliases.Add(o1.Remote(), o2.Remote())- marchAliasLock.Unlock()+ b.march.marchAliasLock.Unlock()b.parse(o1, true)b.parse(o2, false)return isDir(o1)@@ -119,76 +121,76 @@ func (b *bisyncRun) parse(e fs.DirEntry, isPath1 bool) {}func (b *bisyncRun) setupListing() {- ls1 = newFileList()- ls2 = newFileList()+ b.march.ls1 = newFileList()+ b.march.ls2 = newFileList()// note that --ignore-listing-checksum is different from --ignore-checksum// and we already checked it when we set b.opt.Compare.HashType1 and 2- ls1.hash = b.opt.Compare.HashType1- ls2.hash = b.opt.Compare.HashType2+ b.march.ls1.hash = b.opt.Compare.HashType1+ b.march.ls2.hash = b.opt.Compare.HashType2}func (b *bisyncRun) ForObject(o fs.Object, isPath1 bool) {- tr := accounting.Stats(marchCtx).NewCheckingTransfer(o, "listing file - "+whichPath(isPath1))+ tr := accounting.Stats(b.march.marchCtx).NewCheckingTransfer(o, "listing file - "+whichPath(isPath1))defer func() {- tr.Done(marchCtx, nil)+ tr.Done(b.march.marchCtx, nil)}()var (hashVal stringhashErr error)- ls := whichLs(isPath1)+ ls := b.whichLs(isPath1)hashType := ls.hashif hashType != hash.None {- hashVal, hashErr = o.Hash(marchCtx, hashType)- marchErrLock.Lock()- if firstErr == nil {- firstErr = hashErr+ hashVal, hashErr = o.Hash(b.march.marchCtx, hashType)+ b.march.marchErrLock.Lock()+ if b.march.firstErr == nil {+ b.march.firstErr = hashErr}- marchErrLock.Unlock()+ b.march.marchErrLock.Unlock()}- hashVal, hashErr = tryDownloadHash(marchCtx, o, hashVal)- marchErrLock.Lock()- if firstErr == nil {- firstErr = hashErr+ hashVal, hashErr = b.tryDownloadHash(b.march.marchCtx, o, hashVal)+ b.march.marchErrLock.Lock()+ if b.march.firstErr == nil {+ b.march.firstErr = hashErr}- if firstErr != nil {- b.handleErr(hashType, "error hashing during march", firstErr, false, true)+ if b.march.firstErr != nil {+ b.handleErr(hashType, "error hashing during march", b.march.firstErr, false, true)}- marchErrLock.Unlock()+ b.march.marchErrLock.Unlock()var modtime time.Timeif b.opt.Compare.Modtime {- modtime = o.ModTime(marchCtx).In(TZ)+ modtime = o.ModTime(b.march.marchCtx).In(TZ)}id := "" // TODO: ID(o)flags := "-" // "-" for a file and "d" for a directory- marchLsLock.Lock()+ b.march.marchLsLock.Lock()ls.put(o.Remote(), o.Size(), modtime, hashVal, id, flags)- marchLsLock.Unlock()+ b.march.marchLsLock.Unlock()}func (b *bisyncRun) ForDir(o fs.Directory, isPath1 bool) {- tr := accounting.Stats(marchCtx).NewCheckingTransfer(o, "listing dir - "+whichPath(isPath1))+ tr := accounting.Stats(b.march.marchCtx).NewCheckingTransfer(o, "listing dir - "+whichPath(isPath1))defer func() {- tr.Done(marchCtx, nil)+ tr.Done(b.march.marchCtx, nil)}()- ls := whichLs(isPath1)+ ls := b.whichLs(isPath1)var modtime time.Timeif b.opt.Compare.Modtime {- modtime = o.ModTime(marchCtx).In(TZ)+ modtime = o.ModTime(b.march.marchCtx).In(TZ)}id := "" // TODOflags := "d" // "-" for a file and "d" for a directory- marchLsLock.Lock()+ b.march.marchLsLock.Lock()ls.put(o.Remote(), -1, modtime, "", id, flags)- marchLsLock.Unlock()+ b.march.marchLsLock.Unlock()}-func whichLs(isPath1 bool) *fileList {- ls := ls1+func (b *bisyncRun) whichLs(isPath1 bool) *fileList {+ ls := b.march.ls1if !isPath1 {- ls = ls2+ ls = b.march.ls2}return ls}@@ -206,7 +208,7 @@ func (b *bisyncRun) findCheckFiles(ctx context.Context) (*fileList, *fileList, eb.handleErr(b.opt.CheckFilename, "error adding CheckFilename to filter", filterCheckFile.Add(true, b.opt.CheckFilename), true, true)b.handleErr(b.opt.CheckFilename, "error adding ** exclusion to filter", filterCheckFile.Add(false, "**"), true, true)ci := fs.GetConfig(ctxCheckFile)- marchCtx = ctxCheckFile+ b.march.marchCtx = ctxCheckFileb.setupListing()fs.Debugf(b, "starting to march!")@@ -223,18 +225,18 @@ func (b *bisyncRun) findCheckFiles(ctx context.Context) (*fileList, *fileList, eNoCheckDest: false,NoUnicodeNormalization: ci.NoUnicodeNormalization,}- err = m.Run(ctxCheckFile)+ b.march.err = m.Run(ctxCheckFile)- fs.Debugf(b, "march completed. err: %v", err)- if err == nil {- err = firstErr+ fs.Debugf(b, "march completed. err: %v", b.march.err)+ if b.march.err == nil {+ b.march.err = b.march.firstErr}- if err != nil {- b.handleErr("march", "error during findCheckFiles", err, true, true)+ if b.march.err != nil {+ b.handleErr("march", "error during findCheckFiles", b.march.err, true, true)b.abort = true}- return ls1, ls2, err+ return b.march.ls1, b.march.ls2, b.march.err}// ID returns the ID of the Object if known, or "" if notdiff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 6af40a2e2..e458fff60 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -51,6 +51,11 @@ type bisyncRun struct {lockFile stringrenames renamesresyncIs1to2 bool+ march bisyncMarch+ check bisyncCheck+ queueOpt bisyncQueueOpt+ downloadHashOpt downloadHashOpt+ lockFileOpt lockFileOpt}type queues struct {@@ -64,7 +69,6 @@ type queues struct {// Bisync handles lock file, performs bisync run and checks exit statusfunc Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {- defer resetGlobals()opt := *optArg // ensure that input is never changedb := &bisyncRun{fs1: fs1,@@ -124,6 +128,8 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {return err}+ b.queueOpt.logger = operations.NewLoggerOpt()+// Handle SIGINTvar finaliseOnce gosync.Once@@ -161,7 +167,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {markFailed(b.listing1)markFailed(b.listing2)}- b.removeLockFile()+ err = b.removeLockFile()}})}@@ -171,7 +177,10 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {// run bisyncerr = b.runLocked(ctx)- b.removeLockFile()+ removeLockErr := b.removeLockFile()+ if err == nil {+ err = removeLockErr+ }b.CleanupCompleted = trueif b.InGracefulShutdown {@@ -297,7 +306,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {}fs.Infof(nil, "Building Path1 and Path2 listings")- ls1, ls2, err = b.makeMarchListing(fctx)+ b.march.ls1, b.march.ls2, err = b.makeMarchListing(fctx)if err != nil || accounting.Stats(fctx).Errored() {fs.Error(nil, Color(terminal.RedFg, "There were errors while building listings. Aborting as it is too dangerous to continue."))b.critical = true@@ -307,7 +316,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {// Check for Path1 deltas relative to the prior syncfs.Infof(nil, "Path1 checking for diffs")- ds1, err := b.findDeltas(fctx, b.fs1, b.listing1, ls1, "Path1")+ ds1, err := b.findDeltas(fctx, b.fs1, b.listing1, b.march.ls1, "Path1")if err != nil {return err}@@ -315,7 +324,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {// Check for Path2 deltas relative to the prior syncfs.Infof(nil, "Path2 checking for diffs")- ds2, err := b.findDeltas(fctx, b.fs2, b.listing2, ls2, "Path2")+ ds2, err := b.findDeltas(fctx, b.fs2, b.listing2, b.march.ls2, "Path2")if err != nil {return err}@@ -389,7 +398,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {newl1, _ := b.loadListing(b.newListing1)newl2, _ := b.loadListing(b.newListing2)b.debug(b.DebugName, fmt.Sprintf("pre-saveOldListings, ls1 has name?: %v, ls2 has name?: %v", l1.has(b.DebugName), l2.has(b.DebugName)))- b.debug(b.DebugName, fmt.Sprintf("pre-saveOldListings, newls1 has name?: %v, newls2 has name?: %v", newl1.has(b.DebugName), newl2.has(b.DebugName)))+ b.debug(b.DebugName, fmt.Sprintf("pre-saveOldListings, newls1 has name?: %v, ls2 has name?: %v", newl1.has(b.DebugName), newl2.has(b.DebugName)))}b.saveOldListings()// save new listings@@ -553,7 +562,7 @@ func (b *bisyncRun) setBackupDir(ctx context.Context, destPath int) context.Contreturn ctx}-func (b *bisyncRun) overlappingPathsCheck(fctx context.Context, fs1, fs2 fs.Fs) error {+func (b *bisyncRun) overlappingPathsCheck(fctx context.Context, fs1, fs2 fs.Fs) (err error) {if operations.OverlappingFilterCheck(fctx, fs2, fs1) {err = errors.New(Color(terminal.RedFg, "Overlapping paths detected. Cannot bisync between paths that overlap, unless excluded by filters."))return err@@ -586,7 +595,7 @@ func (b *bisyncRun) overlappingPathsCheck(fctx context.Context, fs1, fs2 fs.Fs)return nil}-func (b *bisyncRun) checkSyntax() error {+func (b *bisyncRun) checkSyntax() (err error) {// check for odd number of quotes in path, usually indicating an escaping issuepath1 := bilib.FsPath(b.fs1)path2 := bilib.FsPath(b.fs2)@@ -634,25 +643,3 @@ func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) {}return false}--// mainly to make sure tests don't interfere with each other when running more than one-func resetGlobals() {- downloadHash = false- logger = operations.NewLoggerOpt()- ignoreListingChecksum = false- ignoreListingModtime = false- hashTypes = nil- queueCI = nil- hashType = 0- fsrc, fdst = nil, nil- fcrypt = nil- Opt = Options{}- once = gosync.Once{}- downloadHashWarn = gosync.Once{}- firstDownloadHash = gosync.Once{}- ls1 = newFileList()- ls2 = newFileList()- err = nil- firstErr = nil- marchCtx = nil-}diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex e299f80f7..f618619ff 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -51,19 +51,19 @@ func (rs *ResultsSlice) has(name string) bool {return false}-var (- logger = operations.NewLoggerOpt()+type bisyncQueueOpt struct {+ logger operations.LoggerOptlock mutex.Mutexonce mutex.OnceignoreListingChecksum boolignoreListingModtime boolhashTypes map[string]hash.TypequeueCI *fs.ConfigInfo-)+}// allows us to get the right hashtype during the LoggerFn without knowing whether it's Path1/Path2-func getHashType(fname string) hash.Type {- ht, ok := hashTypes[fname]+func (b *bisyncRun) getHashType(fname string) hash.Type {+ ht, ok := b.queueOpt.hashTypes[fname]if ok {return ht}@@ -106,9 +106,9 @@ func altName(name string, src, dst fs.DirEntry) string {}// WriteResults is Bisync's LoggerFn-func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {- lock.Lock()- defer lock.Unlock()+func (b *bisyncRun) WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {+ b.queueOpt.lock.Lock()+ defer b.queueOpt.lock.Unlock()opt := operations.GetLoggerOpt(ctx)result := Results{@@ -131,14 +131,14 @@ func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEnresult.Flags = "-"if side != nil {result.Size = side.Size()- if !ignoreListingModtime {+ if !b.queueOpt.ignoreListingModtime {result.Modtime = side.ModTime(ctx).In(TZ)}- if !ignoreListingChecksum {+ if !b.queueOpt.ignoreListingChecksum {sideObj, ok := side.(fs.ObjectInfo)if ok {- result.Hash, _ = sideObj.Hash(ctx, getHashType(sideObj.Fs().Name()))- result.Hash, _ = tryDownloadHash(ctx, sideObj, result.Hash)+ result.Hash, _ = sideObj.Hash(ctx, b.getHashType(sideObj.Fs().Name()))+ result.Hash, _ = b.tryDownloadHash(ctx, sideObj, result.Hash)}}@@ -159,8 +159,8 @@ func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEn}prettyprint(result, "writing result", fs.LogLevelDebug)- if result.Size < 0 && result.Flags != "d" && ((queueCI.CheckSum && !downloadHash) || queueCI.SizeOnly) {- once.Do(func() {+ if result.Size < 0 && result.Flags != "d" && ((b.queueOpt.queueCI.CheckSum && !b.downloadHashOpt.downloadHash) || b.queueOpt.queueCI.SizeOnly) {+ b.queueOpt.once.Do(func() {fs.Log(result.Name, Color(terminal.YellowFg, "Files of unknown size (such as Google Docs) do not sync reliably with --checksum or --size-only. Consider using modtime instead (the default) or --drive-skip-gdocs"))})}@@ -189,14 +189,14 @@ func ReadResults(results io.Reader) []Results {// for setup code shared by both fastCopy and resyncDirfunc (b *bisyncRun) preCopy(ctx context.Context) context.Context {- queueCI = fs.GetConfig(ctx)- ignoreListingChecksum = b.opt.IgnoreListingChecksum- ignoreListingModtime = !b.opt.Compare.Modtime- hashTypes = map[string]hash.Type{+ b.queueOpt.queueCI = fs.GetConfig(ctx)+ b.queueOpt.ignoreListingChecksum = b.opt.IgnoreListingChecksum+ b.queueOpt.ignoreListingModtime = !b.opt.Compare.Modtime+ b.queueOpt.hashTypes = map[string]hash.Type{b.fs1.Name(): b.opt.Compare.HashType1,b.fs2.Name(): b.opt.Compare.HashType2,}- logger.LoggerFn = WriteResults+ b.queueOpt.logger.LoggerFn = b.WriteResultsoverridingEqual := falseif (b.opt.Compare.Modtime && b.opt.Compare.Checksum) || b.opt.Compare.DownloadHash {overridingEqual = true@@ -209,15 +209,15 @@ func (b *bisyncRun) preCopy(ctx context.Context) context.Context {fs.Debugf(nil, "overriding equal")ctx = b.EqualFn(ctx)}- ctxCopyLogger := operations.WithSyncLogger(ctx, logger)+ ctxCopyLogger := operations.WithSyncLogger(ctx, b.queueOpt.logger)if b.opt.Compare.Checksum && (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.opt.Compare.SlowHashDetected {// set here in case !b.opt.Compare.Modtime- queueCI = fs.GetConfig(ctxCopyLogger)+ b.queueOpt.queueCI = fs.GetConfig(ctxCopyLogger)if b.opt.Compare.NoSlowHash {- queueCI.CheckSum = false+ b.queueOpt.queueCI.CheckSum = false}if b.opt.Compare.SlowHashSyncOnly && !overridingEqual {- queueCI.CheckSum = true+ b.queueOpt.queueCI.CheckSum = true}}return ctxCopyLogger@@ -250,9 +250,9 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.ctxCopy, b.CancelSync = context.WithCancel(ctxCopy)b.testFn()err := sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)- prettyprint(logger, "logger", fs.LogLevelDebug)+ prettyprint(b.queueOpt.logger, "b.queueOpt.logger", fs.LogLevelDebug)- getResults := ReadResults(logger.JSON)+ getResults := ReadResults(b.queueOpt.logger.JSON)fs.Debugf(nil, "Got %v results for %v", len(getResults), queueName)lineFormat := "%s %8d %s %s %s %q\n"@@ -292,9 +292,9 @@ func (b *bisyncRun) resyncDir(ctx context.Context, fsrc, fdst fs.Fs) ([]Results,ctx = b.preCopy(ctx)err := sync.CopyDir(ctx, fdst, fsrc, b.opt.CreateEmptySrcDirs)- prettyprint(logger, "logger", fs.LogLevelDebug)+ prettyprint(b.queueOpt.logger, "b.queueOpt.logger", fs.LogLevelDebug)- getResults := ReadResults(logger.JSON)+ getResults := ReadResults(b.queueOpt.logger.JSON)fs.Debugf(nil, "Got %v results for %v", len(getResults), "resync")return getResults, errdiff --git a/cmd/bisync/resolve.go b/cmd/bisync/resolve.goindex 21624580d..d40ddb5e6 100644--- a/cmd/bisync/resolve.go+++ b/cmd/bisync/resolve.go@@ -135,7 +135,7 @@ type namePair struct {newName string}-func (b *bisyncRun) resolve(ctxMove context.Context, path1, path2, file, alias string, renameSkipped, copy1to2, copy2to1 *bilib.Names, ds1, ds2 *deltaSet) error {+func (b *bisyncRun) resolve(ctxMove context.Context, path1, path2, file, alias string, renameSkipped, copy1to2, copy2to1 *bilib.Names, ds1, ds2 *deltaSet) (err error) {winningPath := 0if b.opt.ConflictResolve != PreferNone {winningPath = b.conflictWinner(ds1, ds2, file, alias)@@ -261,15 +261,15 @@ func (ri *renamesInfo) getNames(is1to2 bool) (srcOldName, srcNewName, dstOldNamefunc (b *bisyncRun) numerate(ctx context.Context, startnum int, file, alias string) int {for i := startnum; i < math.MaxInt; i++ {iStr := fmt.Sprint(i)- if !ls1.has(SuffixName(ctx, file, b.opt.ConflictSuffix1+iStr)) &&- !ls1.has(SuffixName(ctx, alias, b.opt.ConflictSuffix1+iStr)) &&- !ls2.has(SuffixName(ctx, file, b.opt.ConflictSuffix2+iStr)) &&- !ls2.has(SuffixName(ctx, alias, b.opt.ConflictSuffix2+iStr)) {+ if !b.march.ls1.has(SuffixName(ctx, file, b.opt.ConflictSuffix1+iStr)) &&+ !b.march.ls1.has(SuffixName(ctx, alias, b.opt.ConflictSuffix1+iStr)) &&+ !b.march.ls2.has(SuffixName(ctx, file, b.opt.ConflictSuffix2+iStr)) &&+ !b.march.ls2.has(SuffixName(ctx, alias, b.opt.ConflictSuffix2+iStr)) {// make sure it still holds true with suffixes switched (it should)- if !ls1.has(SuffixName(ctx, file, b.opt.ConflictSuffix2+iStr)) &&- !ls1.has(SuffixName(ctx, alias, b.opt.ConflictSuffix2+iStr)) &&- !ls2.has(SuffixName(ctx, file, b.opt.ConflictSuffix1+iStr)) &&- !ls2.has(SuffixName(ctx, alias, b.opt.ConflictSuffix1+iStr)) {+ if !b.march.ls1.has(SuffixName(ctx, file, b.opt.ConflictSuffix2+iStr)) &&+ !b.march.ls1.has(SuffixName(ctx, alias, b.opt.ConflictSuffix2+iStr)) &&+ !b.march.ls2.has(SuffixName(ctx, file, b.opt.ConflictSuffix1+iStr)) &&+ !b.march.ls2.has(SuffixName(ctx, alias, b.opt.ConflictSuffix1+iStr)) {fs.Debugf(file, "The first available suffix is: %s", iStr)return i}@@ -280,10 +280,10 @@ func (b *bisyncRun) numerate(ctx context.Context, startnum int, file, alias stri// like numerate, but consider only one side's suffix (for when suffixes are different)func (b *bisyncRun) numerateSingle(ctx context.Context, startnum int, file, alias string, path int) int {- lsA, lsB := ls1, ls2+ lsA, lsB := b.march.ls1, b.march.ls2suffix := b.opt.ConflictSuffix1if path == 2 {- lsA, lsB = ls2, ls1+ lsA, lsB = b.march.ls2, b.march.ls1suffix = b.opt.ConflictSuffix2}for i := startnum; i < math.MaxInt; i++ {@@ -299,7 +299,7 @@ func (b *bisyncRun) numerateSingle(ctx context.Context, startnum int, file, aliareturn 0 // not really possible, as no one has 9223372036854775807 conflicts, and if they do, they have bigger problems}-func (b *bisyncRun) rename(ctx context.Context, thisNamePair namePair, thisPath, thatPath string, thisFs fs.Fs, thisPathNum, thatPathNum, winningPath int, q, renameSkipped *bilib.Names) error {+func (b *bisyncRun) rename(ctx context.Context, thisNamePair namePair, thisPath, thatPath string, thisFs fs.Fs, thisPathNum, thatPathNum, winningPath int, q, renameSkipped *bilib.Names) (err error) {if winningPath == thisPathNum {b.indent(fmt.Sprintf("!Path%d", thisPathNum), thisPath+thisNamePair.newName, fmt.Sprintf("Not renaming Path%d copy, as it was determined the winner", thisPathNum))} else {@@ -321,7 +321,7 @@ func (b *bisyncRun) rename(ctx context.Context, thisNamePair namePair, thisPath,return nil}-func (b *bisyncRun) delete(ctx context.Context, thisNamePair namePair, thisPath, thatPath string, thisFs fs.Fs, thisPathNum, thatPathNum int, renameSkipped *bilib.Names) error {+func (b *bisyncRun) delete(ctx context.Context, thisNamePair namePair, thisPath, thatPath string, thisFs fs.Fs, thisPathNum, thatPathNum int, renameSkipped *bilib.Names) (err error) {skip := operations.SkipDestructive(ctx, thisNamePair.oldName, "delete")if !skip {b.indent(fmt.Sprintf("!Path%d", thisPathNum), thisPath+thisNamePair.oldName, fmt.Sprintf("Deleting Path%d copy", thisPathNum))diff --git a/cmd/bisync/resync.go b/cmd/bisync/resync.goindex 4606a43e1..6df61089a 100644--- a/cmd/bisync/resync.go+++ b/cmd/bisync/resync.go@@ -41,12 +41,12 @@ func (b *bisyncRun) setResyncDefaults() {// It will generate path1 and path2 listings,// copy any unique files to the opposite path,// and resolve any differing files according to the --resync-mode.-func (b *bisyncRun) resync(octx, fctx context.Context) error {+func (b *bisyncRun) resync(octx, fctx context.Context) (err error) {fs.Infof(nil, "Copying Path2 files to Path1")// Save blank filelists (will be filled from sync results)- var ls1 = newFileList()- var ls2 = newFileList()+ ls1 := newFileList()+ ls2 := newFileList()err = ls1.save(fctx, b.newListing1)if err != nil {b.handleErr(ls1, "error saving ls1 from resync", err, true, true)diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex e36988c37..957314f44 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -1815,6 +1815,9 @@ about _Unison_ and synchronization in general.## Changelog+### `v1.71`+* Fixed an issue causing errors when running concurrent bisync runs through the `rc`.+### `v1.69.1`* Fixed an issue causing listings to not capture concurrent modifications under certain conditions