"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fstests: check backends return accurate object fingerprints after writes
Nick Craig-Wood7dd42c93cf25
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.goindex 52f44153b..c03db1749 100644--- a/fstest/fstests/fstests.go+++ b/fstest/fstests/fstests.go@@ -146,7 +146,22 @@ func retry(t *testing.T, what string, f func() error) {require.NoError(t, err, what)}-// check interface+// checkFingerprint checks that the fingerprint of the in-memory+// object (memory) exactly matches the fingerprint of the same object+// read back from the remote (reloaded).+func checkFingerprint(ctx context.Context, t *testing.T, memory, reloaded fs.Object) {+ t.Helper()+ for _, test := range []struct {+ fast bool+ name string+ }{+ {fast: false, name: "slow"},+ {fast: true, name: "fast"},+ } {+ memoryFingerprint, reloadedFingerprint := fs.Fingerprint(ctx, memory, test.fast), fs.Fingerprint(ctx, reloaded, test.fast)+ assert.Equal(t, memoryFingerprint, reloadedFingerprint, "%s fingerprint of object in memory (%v) should exactly match fingerprint read back from the remote (%v)", test.name, memoryFingerprint, reloadedFingerprint)+ }+}// PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove//@@ -198,8 +213,10 @@ func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *f}file.Check(t, obj, f.Precision())// Re-read the object and check again+ liveObj := objobj = fstest.NewObject(ctx, t, f, file.Path)file.Check(t, obj, f.Precision())+ checkFingerprint(ctx, t, liveObj, obj)}return obj}@@ -253,8 +270,10 @@ func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item,file.Check(t, obj, f.Precision())// Re-read the object and check again+ liveObj := objobj = fstest.NewObject(ctx, t, f, file.Path)file.Check(t, obj, f.Precision())+ checkFingerprint(ctx, t, liveObj, obj)// Download the object and check it is OKdownloadHash := hash.NewMultiHasher()@@ -1276,6 +1295,7 @@ func Run(t *testing.T, opt *Opt) {// Check dst lightly - list above has checked ModTime/Hashesassert.Equal(t, file2Copy.Path, dst.Remote())+ checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Copy.Path))// check that mutating dst does not mutate srcif !strings.Contains(fs.ConfigStringFull(f), "copy_is_hardlink") {@@ -1332,8 +1352,10 @@ func Run(t *testing.T, opt *Opt) {// Check metadata is correctfstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)+ liveObj := oDstoDst = fstest.NewObject(ctx, t, f, dstName)fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)+ checkFingerprint(ctx, t, liveObj, oDst)// Remove test filesrequire.NoError(t, oSrc.Remove(ctx))@@ -1371,6 +1393,7 @@ func Run(t *testing.T, opt *Opt) {fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})// Check dst lightly - list above has checked ModTime/Hashesassert.Equal(t, file2Move.Path, dst.Remote())+ checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Move.Path))// 1: file name.txt// 2: other.txt@@ -1440,8 +1463,10 @@ func Run(t *testing.T, opt *Opt) {// Check metadata is correctfstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)+ liveObj := newOnewO = fstest.NewObject(ctx, t, f, newName)fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)+ checkFingerprint(ctx, t, liveObj, newO)// Remove test filerequire.NoError(t, newO.Remove(ctx))@@ -1763,6 +1788,7 @@ func Run(t *testing.T, opt *Opt) {require.NoError(t, err)file1.ModTime = newModTimefile1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())+ checkFingerprint(ctx, t, obj, fstest.NewObject(ctx, t, f, file1.Path))// And make a new object and read it from there tooTestObjectModTime(t)})@@ -1827,33 +1853,34 @@ func Run(t *testing.T, opt *Opt) {obj := fstest.NewObject(ctx, t, f, file1.Path)// Read the whole object first so any lazily fetched hash- // is populated, then take that as the reference.+ // is populated, then read a reference object to compare+ // fingerprints against._ = ReadObject(ctx, t, obj, -1)- fastWant := fs.Fingerprint(ctx, obj, true)- slowWant := fs.Fingerprint(ctx, obj, false)-- checkFingerprint := func(what string) {- assert.Equal(t, fastWant, fs.Fingerprint(ctx, obj, true), "fast fingerprint changed after %s", what)- assert.Equal(t, slowWant, fs.Fingerprint(ctx, obj, false), "slow fingerprint changed after %s", what)- }+ reference := fstest.NewObject(ctx, t, f, file1.Path)+ _ = ReadObject(ctx, t, reference, -1)// Opening the object in various ways must not change the fingerprint- _ = ReadObject(ctx, t, obj, -1)- checkFingerprint("full read")-- _ = ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50})- checkFingerprint("seek read")-- _ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 5, End: 15})- checkFingerprint("range read")-- _ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 80, End: -1})- checkFingerprint("range read to end")+ for _, test := range []struct {+ name string+ options []fs.OpenOption+ }{+ {name: "full read"},+ {name: "seek read", options: []fs.OpenOption{&fs.SeekOption{Offset: 50}}},+ {name: "range read", options: []fs.OpenOption{&fs.RangeOption{Start: 5, End: 15}}},+ {name: "range read to end", options: []fs.OpenOption{&fs.RangeOption{Start: 80, End: -1}}},+ } {+ t.Run(test.name, func(t *testing.T) {+ _ = ReadObject(ctx, t, obj, -1, test.options...)+ checkFingerprint(ctx, t, obj, reference)+ })+ }// A freshly fetched object must have the same fingerprint too- obj = fstest.NewObject(ctx, t, f, file1.Path)- _ = ReadObject(ctx, t, obj, -1)- checkFingerprint("refresh")+ t.Run("refresh", func(t *testing.T) {+ obj := fstest.NewObject(ctx, t, f, file1.Path)+ _ = ReadObject(ctx, t, obj, -1)+ checkFingerprint(ctx, t, obj, reference)+ })})// TestObjectUpdate tests that Update works@@ -1880,8 +1907,10 @@ func Run(t *testing.T, opt *Opt) {file1.Check(t, obj, f.Precision())// Re-read the object and check again+ liveObj := objobj = fstest.NewObject(ctx, t, f, file1.Path)file1.Check(t, obj, f.Precision())+ checkFingerprint(ctx, t, liveObj, obj)// check contents correctassert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ")@@ -2377,8 +2406,10 @@ func Run(t *testing.T, opt *Opt) {file.Size = int64(contentSize) // use correct size when checkingfile.Check(t, obj, f.Precision())// Re-read the object and check again+ liveObj := objobj = fstest.NewObject(ctx, t, f, file.Path)file.Check(t, obj, f.Precision())+ checkFingerprint(ctx, t, liveObj, obj)require.NoError(t, obj.Remove(ctx))})}