"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: fall back to --download when a checksum is missing
nielashc6749ff70ef6
diff --git a/cmd/bisync/checkfn.go b/cmd/bisync/checkfn.goindex 162c0f3ed..e6694a458 100644--- a/cmd/bisync/checkfn.go+++ b/cmd/bisync/checkfn.go@@ -80,7 +80,7 @@ func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool,return true, false, err}if ht == hash.None {- return false, true, nil+ return missingHashFn(ctx, dst, src)}if !same {err = fmt.Errorf("%v differ", ht)@@ -99,14 +99,14 @@ func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (differeturn true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)}if underlyingHash == "" {- return false, true, nil+ return missingHashFn(ctx, dst, src)}cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)if err != nil {return true, false, fmt.Errorf("error computing hash: %w", err)}if cryptHash == "" {- return false, true, nil+ return missingHashFn(ctx, dst, src)}if cryptHash != underlyingHash {err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)@@ -125,6 +125,15 @@ func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object)return b.CryptCheckFn(ctx, src, dst)}+func missingHashFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {+ ci := fs.GetConfig(ctx)+ if ci.SizeOnly || ci.IgnoreChecksum {+ return false, true, nil+ }+ fs.Debugf(src, "hash is missing, so using --download")+ return DownloadCheckFn(ctx, dst, src)+}+// DownloadCheckFn is a slightly modified version of Check with --downloadfunc DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHash bool, err error) {equal, err = operations.CheckIdenticalDownload(ctx, src, dst)diff --git a/cmd/bisync/checkfn_internal_test.go b/cmd/bisync/checkfn_internal_test.gonew file mode 100644index 000000000..6a194f19c--- /dev/null+++ b/cmd/bisync/checkfn_internal_test.go@@ -0,0 +1,59 @@+package bisync++import (+ "context"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fstest/mockfs"+ "github.com/rclone/rclone/fstest/mockobject"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// blankHashObject supports MD5 through its Fs but returns a blank hash+type blankHashObject struct {+ *mockobject.ContentMockObject+}++func (o blankHashObject) Hash(ctx context.Context, t hash.Type) (string, error) {+ return "", nil+}++func newBlankHashObject(ctx context.Context, t *testing.T, fsName, content string) blankHashObject {+ f, err := mockfs.NewFs(ctx, fsName, "", nil)+ require.NoError(t, err)+ f.(*mockfs.Fs).SetHashes(hash.NewHashSet(hash.MD5))+ o := mockobject.New("file.txt").WithContent([]byte(content), mockobject.SeekModeNone)+ o.SetFs(f)+ return blankHashObject{o}+}++func TestCheckFnMissingHash(t *testing.T) {+ for _, test := range []struct {+ name string+ content string+ flag func(ci *fs.ConfigInfo)+ wantDiffer bool+ wantNoHash bool+ }{+ {name: "identical", content: "hello"},+ {name: "same size, different content", content: "HELLO", wantDiffer: true},+ {name: "size-only", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantNoHash: true},+ {name: "ignore-checksum", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantNoHash: true},+ } {+ t.Run(test.name, func(t *testing.T) {+ ctx, ci := fs.AddConfig(context.Background())+ if test.flag != nil {+ test.flag(ci)+ }+ src := newBlankHashObject(ctx, t, "path1", "hello")+ dst := newBlankHashObject(ctx, t, "path2", test.content)+ differ, noHash, err := CheckFn(ctx, dst, src)+ require.NoError(t, err)+ assert.Equal(t, test.wantDiffer, differ, "differ")+ assert.Equal(t, test.wantNoHash, noHash, "noHash")+ })+ }+}diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex 1e4475a0d..737397827 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -1941,6 +1941,9 @@ the `--max-delete` safety check when used with `--track-renames`.Path2 won.- Fixed an issue causing `--conflict-resolve` to compare a zero size or modtime.- Fixed an issue causing an endless retry loop for certain skipped files.+- When a file changed on both sides has an unexpectedly blank hash (for example,+an S3 multipart object without an MD5), bisync now compares it by downloading+it, as with `--download`, instead of by size alone.### `v1.74.2`