"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: fallback to cryptcheck or --download when can't check hash
nielash422b03708703
diff --git a/cmd/bisync/checkfn.go b/cmd/bisync/checkfn.gonew file mode 100644index 000000000..cc462772b--- /dev/null+++ b/cmd/bisync/checkfn.go@@ -0,0 +1,191 @@+package bisync++import (+ "bytes"+ "context"+ "fmt"+ "strings"++ "github.com/rclone/rclone/backend/crypt"+ "github.com/rclone/rclone/cmd/bisync/bilib"+ "github.com/rclone/rclone/cmd/check"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/filter"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/operations"+)++var hashType hash.Type+var fsrc, fdst fs.Fs+var fcrypt *crypt.Fs++// WhichCheck determines which CheckFn we should use based on the Fs types+// It is more robust and accurate than Check because+// it will fallback to CryptCheck or DownloadCheck instead of --size-only!+// it returns the *operations.CheckOpt with the CheckFn set.+func WhichCheck(ctx context.Context, opt *operations.CheckOpt) *operations.CheckOpt {+ ci := fs.GetConfig(ctx)+ common := opt.Fsrc.Hashes().Overlap(opt.Fdst.Hashes())++ // note that ci.IgnoreChecksum doesn't change the behavior of Check -- it's just a way to opt-out of cryptcheck/download+ if common.Count() > 0 || ci.SizeOnly || ci.IgnoreChecksum {+ // use normal check+ opt.Check = CheckFn+ return opt+ }++ FsrcCrypt, srcIsCrypt := opt.Fsrc.(*crypt.Fs)+ FdstCrypt, dstIsCrypt := opt.Fdst.(*crypt.Fs)++ if (srcIsCrypt && dstIsCrypt) || (!srcIsCrypt && dstIsCrypt) {+ // if both are crypt or only dst is crypt+ hashType = FdstCrypt.UnWrap().Hashes().GetOne()+ if hashType != hash.None {+ // use cryptcheck+ fsrc = opt.Fsrc+ fdst = opt.Fdst+ fcrypt = FdstCrypt+ fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")+ opt.Check = CryptCheckFn+ return opt+ }+ } else if srcIsCrypt && !dstIsCrypt {+ // if only src is crypt+ hashType = FsrcCrypt.UnWrap().Hashes().GetOne()+ if hashType != hash.None {+ // use reverse cryptcheck+ fsrc = opt.Fdst+ fdst = opt.Fsrc+ fcrypt = FsrcCrypt+ fs.Infof(fdst, "Crypt detected! Using cryptcheck instead of check. (Use --size-only or --ignore-checksum to disable)")+ opt.Check = ReverseCryptCheckFn+ return opt+ }+ }++ // if we've gotten this far, niether check or cryptcheck will work, so use --download+ fs.Infof(fdst, "Can't compare hashes, so using check --download for safety. (Use --size-only or --ignore-checksum to disable)")+ opt.Check = DownloadCheckFn+ return opt+}++// CheckFn is a slightly modified version of Check+func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+ same, ht, err := operations.CheckHashes(ctx, src, dst)+ if err != nil {+ return true, false, err+ }+ if ht == hash.None {+ return false, true, nil+ }+ if !same {+ err = fmt.Errorf("%v differ", ht)+ fs.Errorf(src, "%v", err)+ return true, false, nil+ }+ return false, false, nil+}++// CryptCheckFn is a slightly modified version of CryptCheck+func CryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+ cryptDst := dst.(*crypt.Object)+ underlyingDst := cryptDst.UnWrap()+ underlyingHash, err := underlyingDst.Hash(ctx, hashType)+ if err != nil {+ return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)+ }+ if underlyingHash == "" {+ return false, true, nil+ }+ cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)+ if err != nil {+ return true, false, fmt.Errorf("error computing hash: %w", err)+ }+ if cryptHash == "" {+ return false, true, nil+ }+ if cryptHash != underlyingHash {+ err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)+ fs.Errorf(src, err.Error())+ return true, false, nil+ }+ return false, false, nil+}++// ReverseCryptCheckFn is like CryptCheckFn except src and dst are switched+// result: src is crypt, dst is non-crypt+func ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+ return CryptCheckFn(ctx, src, dst)+}++// DownloadCheckFn is a slightly modified version of Check with --download+func DownloadCheckFn(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error) {+ differ, err = operations.CheckIdenticalDownload(ctx, a, b)+ if err != nil {+ return true, true, fmt.Errorf("failed to download: %w", err)+ }+ return differ, false, nil+}++// check potential conflicts (to avoid renaming if already identical)+func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (bilib.Names, error) {+ matches := bilib.Names{}+ if filterCheck.HaveFilesFrom() {+ fs.Debugf(nil, "There are potential conflicts to check.")++ opt, close, checkopterr := check.GetCheckOpt(b.fs1, b.fs2)+ if checkopterr != nil {+ b.critical = true+ b.retryable = true+ fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)+ return matches, checkopterr+ }+ defer close()++ opt.Match = new(bytes.Buffer)++ opt = WhichCheck(ctxCheck, opt)++ fs.Infof(nil, "Checking potential conflicts...")+ check := operations.CheckFn(ctxCheck, opt)+ fs.Infof(nil, "Finished checking the potential conflicts. %s", check)++ //reset error count, because we don't want to count check errors as bisync errors+ accounting.Stats(ctxCheck).ResetErrors()++ //return the list of identical files to check against later+ if len(fmt.Sprint(opt.Match)) > 0 {+ matches = bilib.ToNames(strings.Split(fmt.Sprint(opt.Match), "\n"))+ }+ if matches.NotEmpty() {+ fs.Debugf(nil, "The following potential conflicts were determined to be identical. %v", matches)+ } else {+ fs.Debugf(nil, "None of the conflicts were determined to be identical.")+ }++ }+ return matches, nil+}++// WhichEqual is similar to WhichCheck, but checks a single object.+// Returns true if the objects are equal, false if they differ or if we don't know+func WhichEqual(ctx context.Context, src, dst fs.Object, Fsrc, Fdst fs.Fs) bool {+ opt, close, checkopterr := check.GetCheckOpt(Fsrc, Fdst)+ if checkopterr != nil {+ fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)+ }+ defer close()++ opt = WhichCheck(ctx, opt)+ differ, noHash, err := opt.Check(ctx, dst, src)+ if err != nil {+ fs.Errorf(src, "failed to check: %v", err)+ return false+ }+ if noHash {+ fs.Errorf(src, "failed to check as hash is missing")+ return false+ }+ return !differ+}diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex 23a10540f..d441ca404 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -3,7 +3,6 @@package bisyncimport (- "bytes""context""fmt""path/filepath"@@ -11,9 +10,7 @@ import ("strings""github.com/rclone/rclone/cmd/bisync/bilib"- "github.com/rclone/rclone/cmd/check""github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/operations""golang.org/x/text/unicode/norm"@@ -96,47 +93,6 @@ func (ds *deltaSet) printStats() {ds.msg, nAll, nNew, nNewer, nOlder, nDeleted)}-// check potential conflicts (to avoid renaming if already identical)-func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (bilib.Names, error) {- matches := bilib.Names{}- if filterCheck.HaveFilesFrom() {- fs.Debugf(nil, "There are potential conflicts to check.")-- opt, close, checkopterr := check.GetCheckOpt(b.fs1, b.fs2)- if checkopterr != nil {- b.critical = true- b.retryable = true- fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)- return matches, checkopterr- }- defer close()-- opt.Match = new(bytes.Buffer)-- // TODO: consider using custom CheckFn to act like cryptcheck, if either fs is a crypt remote and -c has been passed- // note that cryptCheck() is not currently exported-- fs.Infof(nil, "Checking potential conflicts...")- check := operations.Check(ctxCheck, opt)- fs.Infof(nil, "Finished checking the potential conflicts. %s", check)-- //reset error count, because we don't want to count check errors as bisync errors- accounting.Stats(ctxCheck).ResetErrors()-- //return the list of identical files to check against later- if len(fmt.Sprint(opt.Match)) > 0 {- matches = bilib.ToNames(strings.Split(fmt.Sprint(opt.Match), "\n"))- }- if matches.NotEmpty() {- fs.Debugf(nil, "The following potential conflicts were determined to be identical. %v", matches)- } else {- fs.Debugf(nil, "None of the conflicts were determined to be identical.")- }-- }- return matches, nil-}-// findDeltasfunc (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string, now *fileList, msg string) (ds *deltaSet, err error) {var old *fileListdiff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.goindex 313569ed8..20f2fa00b 100644--- a/cmd/bisync/listing.go+++ b/cmd/bisync/listing.go@@ -641,7 +641,8 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,fs.Debugf(srcObj, "rechecking")for _, dstObj := range dstObjs {if srcObj.Remote() == dstObj.Remote() || srcObj.Remote() == b.aliases.Alias(dstObj.Remote()) {- if operations.Equal(ctxRecheck, srcObj, dstObj) || b.opt.DryRun {+ // note: unlike Equal(), WhichEqual() does not update the modtime in dest if sums match but modtimes don't.+ if b.opt.DryRun || WhichEqual(ctxRecheck, srcObj, dstObj, src, dst) {putObj(srcObj, src, srcList)putObj(dstObj, dst, dstList)resolved = append(resolved, srcObj.Remote())@@ -655,7 +656,8 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,// skip and error during --resync, as rollback is not possibleif !slices.Contains(resolved, srcObj.Remote()) && !b.opt.DryRun {if b.opt.Resync {- b.handleErr(srcObj, "Unable to rollback during --resync", errors.New("no dstObj match or files not equal"), true, false)+ err = errors.New("no dstObj match or files not equal")+ b.handleErr(srcObj, "Unable to rollback during --resync", err, true, false)} else {toRollback = append(toRollback, srcObj.Remote())}diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-oldindex 0f2ac67fe..6815fa4ee 100644--- a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old@@ -1,5 +1,5 @@# bisync listing v1 from test- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/éééö.txt"- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-newindex 9afa4b347..6f843cfa9 100644--- a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new@@ -1,5 +1,5 @@# bisync listing v1 from test- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/éééö.txt"- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-oldindex a8c82fc73..ed2547754 100644--- a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old@@ -1,5 +1,5 @@# bisync listing v1 from test- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "folder/éééö.txt"- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex f3354dc1e..947128fee 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -1367,6 +1367,8 @@ for performance improvements and less [risk of error](https://forum.rclone.org/t* Fixed handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)* `--resync` is now much more efficient (especially for users of `--create-empty-src-dirs`)* Google Docs (and other files of unknown size) are now supported (with the same options as in `sync`)+* Equality checks before a sync conflict rename now fall back to `cryptcheck` (when possible) or `--download`,+instead of of `--size-only`, when `check` is not available.### `v1.64`* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)