Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bisync: fall back to --download when a checksum is missing

nielashc6749ff70ef6
diff --git a/cmd/bisync/checkfn.go b/cmd/bisync/checkfn.go
index 162c0f3ed..e6694a458 100644
--- a/cmd/bisync/checkfn.go
+++ b/cmd/bisync/checkfn.go
@@ -80,7 +80,7 @@ func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool,
return true, false, err
}
if ht == hash.None {
- return false, true, nil
+ return missingHashFn(ctx, dst, src)
}
if !same {
err = fmt.Errorf("%v differ", ht)
@@ -99,14 +99,14 @@ func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (diffe
return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)
}
if underlyingHash == "" {
- return false, true, nil
+ return missingHashFn(ctx, dst, src)
}
cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)
if err != nil {
return true, false, fmt.Errorf("error computing hash: %w", err)
}
if cryptHash == "" {
- return false, true, nil
+ return missingHashFn(ctx, dst, src)
}
if cryptHash != underlyingHash {
err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)
@@ -125,6 +125,15 @@ func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object)
return b.CryptCheckFn(ctx, src, dst)
}
+func missingHashFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+ ci := fs.GetConfig(ctx)
+ if ci.SizeOnly || ci.IgnoreChecksum {
+ return false, true, nil
+ }
+ fs.Debugf(src, "hash is missing, so using --download")
+ return DownloadCheckFn(ctx, dst, src)
+}
+
// DownloadCheckFn is a slightly modified version of Check with --download
func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHash bool, err error) {
equal, err = operations.CheckIdenticalDownload(ctx, src, dst)
diff --git a/cmd/bisync/checkfn_internal_test.go b/cmd/bisync/checkfn_internal_test.go
new file mode 100644
index 000000000..6a194f19c
--- /dev/null
+++ b/cmd/bisync/checkfn_internal_test.go
@@ -0,0 +1,59 @@
+package bisync
+
+import (
+ "context"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fstest/mockfs"
+ "github.com/rclone/rclone/fstest/mockobject"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// blankHashObject supports MD5 through its Fs but returns a blank hash
+type blankHashObject struct {
+ *mockobject.ContentMockObject
+}
+
+func (o blankHashObject) Hash(ctx context.Context, t hash.Type) (string, error) {
+ return "", nil
+}
+
+func newBlankHashObject(ctx context.Context, t *testing.T, fsName, content string) blankHashObject {
+ f, err := mockfs.NewFs(ctx, fsName, "", nil)
+ require.NoError(t, err)
+ f.(*mockfs.Fs).SetHashes(hash.NewHashSet(hash.MD5))
+ o := mockobject.New("file.txt").WithContent([]byte(content), mockobject.SeekModeNone)
+ o.SetFs(f)
+ return blankHashObject{o}
+}
+
+func TestCheckFnMissingHash(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ content string
+ flag func(ci *fs.ConfigInfo)
+ wantDiffer bool
+ wantNoHash bool
+ }{
+ {name: "identical", content: "hello"},
+ {name: "same size, different content", content: "HELLO", wantDiffer: true},
+ {name: "size-only", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantNoHash: true},
+ {name: "ignore-checksum", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantNoHash: true},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ ctx, ci := fs.AddConfig(context.Background())
+ if test.flag != nil {
+ test.flag(ci)
+ }
+ src := newBlankHashObject(ctx, t, "path1", "hello")
+ dst := newBlankHashObject(ctx, t, "path2", test.content)
+ differ, noHash, err := CheckFn(ctx, dst, src)
+ require.NoError(t, err)
+ assert.Equal(t, test.wantDiffer, differ, "differ")
+ assert.Equal(t, test.wantNoHash, noHash, "noHash")
+ })
+ }
+}
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 1e4475a0d..737397827 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -1941,6 +1941,9 @@ the `--max-delete` safety check when used with `--track-renames`.
Path2 won.
- Fixed an issue causing `--conflict-resolve` to compare a zero size or modtime.
- Fixed an issue causing an endless retry loop for certain skipped files.
+- When a file changed on both sides has an unexpectedly blank hash (for example,
+an S3 multipart object without an MD5), bisync now compares it by downloading
+it, as with `--download`, instead of by size alone.
### `v1.74.2`