"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: fix index tests failing in the integration tests
Nick Craig-Woode14337ebef71
diff --git a/fs/operations/index_test.go b/fs/operations/index_test.goindex 56e057e03..fc82d3fca 100644--- a/fs/operations/index_test.go+++ b/fs/operations/index_test.go@@ -8,6 +8,7 @@ import ("os""path/filepath""sort"+ "strings""testing""time"@@ -56,6 +57,17 @@ func (r *indexRun) index(t *testing.T) (transfers, deletes int64) {return stats.GetTransfers(), stats.GetDeletes()}+// assertCount checks a count of transfers or deletes returned by index+//+// It is not checked on chunker as moving its chunks into place is+// counted in the stats too.+func (r *indexRun) assertCount(t *testing.T, want, got int64, msgAndArgs ...any) {+ if strings.HasPrefix(r.Fremote.Name(), "TestChunker") {+ return+ }+ assert.Equal(t, want, got, msgAndArgs...)+}+// read returns the contents of the object at remotefunc (r *indexRun) read(t *testing.T, remote string) string {o, err := r.Fremote.NewObject(r.ctx, remote)@@ -77,20 +89,22 @@ func (r *indexRun) files(t *testing.T) (names []string) {}// checkMimeType checks the MIME type of the object at remote is want,-// ignoring formatting differences, on backends which store it+// ignoring any parameters, on backends which can both write and read it+//+// The parameters are ignored as many backends don't store them.func (r *indexRun) checkMimeType(t *testing.T, remote, want string) {- o, err := r.Fremote.NewObject(r.ctx, remote)- require.NoError(t, err)- if _, ok := o.(fs.MimeTyper); !ok {+ features := r.Fremote.Features()+ if !features.ReadMimeType || !features.WriteMimeType {return}+ o, err := r.Fremote.NewObject(r.ctx, remote)+ require.NoError(t, err)got := fs.MimeType(r.ctx, o)- wantType, wantParams, err := mime.ParseMediaType(want)+ wantType, _, err := mime.ParseMediaType(want)require.NoError(t, err)- gotType, gotParams, err := mime.ParseMediaType(got)+ gotType, _, err := mime.ParseMediaType(got)require.NoError(t, err, got)assert.Equal(t, wantType, gotType)- assert.Equal(t, wantParams, gotParams)}// checkFiles checks the remote contains exactly the objects named@@ -104,8 +118,8 @@ func TestIndex(t *testing.T) {r.opt.Outputs = []string{"index.html=html", "index.json=json", "caddy.json=caddy"}transfers, deletes := r.index(t)- assert.Equal(t, int64(9), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 9, transfers)+ r.assertCount(t, 0, deletes)r.checkFiles(t,"file1.txt", "index.html", "index.json", "caddy.json","sub/file2.txt", "sub/index.html", "sub/index.json", "sub/caddy.json",@@ -139,26 +153,26 @@ func TestIndex(t *testing.T) {// A second run changes nothingtransfers, deletes = r.index(t)- assert.Equal(t, int64(0), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 0, transfers)+ r.assertCount(t, 0, deletes)// Unless it is told to rewrite everythingr.opt.Rewrite = truetransfers, _ = r.index(t)- assert.Equal(t, int64(9), transfers)+ r.assertCount(t, 9, transfers)r.opt.Rewrite = false// Adding a file only rewrites the listing of its directoryr.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", t1)transfers, deletes = r.index(t)- assert.Equal(t, int64(3), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 3, transfers)+ r.assertCount(t, 0, deletes)assert.Contains(t, r.read(t, "sub/deep/index.html"), "file4.txt")// A run with --dry-run changes nothing+ r.WriteObject(r.ctx, "sub/deep/file5.txt", "hello5", t1)ctx, ci := fs.AddConfig(r.ctx)ci.DryRun = true- r.WriteObject(ctx, "sub/deep/file5.txt", "hello5", t1)require.NoError(t, operations.Index(ctx, r.Fremote, &r.opt))assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file5.txt")}@@ -176,15 +190,15 @@ func TestIndexDelete(t *testing.T) {transfers, deletes := r.index(t)if canHaveEmptyDirectories {// The empty directory still exists so keeps its listing- assert.Equal(t, int64(1), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 1, transfers)+ r.assertCount(t, 0, deletes)r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/index.html")assert.Contains(t, r.read(t, "sub/index.html"), "deep/")assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file3.txt")} else {// The directory only existed because of the listing, so it goes- assert.Equal(t, int64(1), transfers)- assert.Equal(t, int64(1), deletes)+ r.assertCount(t, 1, transfers)+ r.assertCount(t, 1, deletes)r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html")assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")@@ -193,8 +207,8 @@ func TestIndexDelete(t *testing.T) {require.NoError(t, err)require.NoError(t, o.Remove(r.ctx))transfers, deletes = r.index(t)- assert.Equal(t, int64(1), transfers)- assert.Equal(t, int64(1), deletes)+ r.assertCount(t, 1, transfers)+ r.assertCount(t, 1, deletes)r.checkFiles(t, "file1.txt", "index.html")assert.NotContains(t, r.read(t, "index.html"), "sub/")}@@ -224,8 +238,8 @@ func TestIndexFilter(t *testing.T) {// A second run changes nothing, in particular it doesn't delete// other/index.html even though other/ has no visible contenttransfers, deletes := r.index(t)- assert.Equal(t, int64(0), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 0, transfers)+ r.assertCount(t, 0, deletes)}func TestIndexRules(t *testing.T) {@@ -246,7 +260,7 @@ func TestIndexRules(t *testing.T) {require.NoError(t, err)require.NoError(t, o.Remove(r.ctx))transfers, _ := r.index(t)- assert.Equal(t, int64(1), transfers) // sub/deep/index.html+ r.assertCount(t, 1, transfers) // sub/deep/index.htmlr.checkFiles(t, "file1.txt", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html")assert.NotEqual(t, "hand written", r.read(t, "sub/deep/index.html"))}@@ -380,12 +394,12 @@ func TestIndexNoHash(t *testing.T) {r.writeFiles()transfers, _ := r.index(t)- assert.Equal(t, int64(3), transfers)+ r.assertCount(t, 3, transfers)transfers, _ = r.index(t)- assert.Equal(t, int64(0), transfers)+ r.assertCount(t, 0, transfers)r.WriteObject(r.ctx, "sub/file4.txt", "hello4", t1)transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/index.html"), "file4.txt")}@@ -399,7 +413,7 @@ func TestIndexChanged(t *testing.T) {r.WriteObject(r.ctx, "other/undeclared.txt", "hello", t1)r.opt.Changed = []string{"sub/deep/file4.txt"}transfers, _ := r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/deep/index.html"), "file4.txt")assert.NotContains(t, r.read(t, "other/index.html"), "undeclared.txt")@@ -407,7 +421,7 @@ func TestIndexChanged(t *testing.T) {r.WriteObject(r.ctx, "sub/new/file5.txt", "hello5", t1)r.opt.Changed = []string{"sub/new/file5.txt"}transfers, _ = r.index(t)- assert.Equal(t, int64(2), transfers)+ r.assertCount(t, 2, transfers)assert.Contains(t, r.read(t, "sub/index.html"), `<a href="./new/">new/</a>`)assert.Contains(t, r.read(t, "sub/new/index.html"), "file5.txt")@@ -417,7 +431,7 @@ func TestIndexChanged(t *testing.T) {r.WriteObject(r.ctx, name, "hello", t1)r.opt.Changed = []string{changed}transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers, changed)+ r.assertCount(t, 1, transfers, changed)assert.Contains(t, r.read(t, "other/index.html"), name[6:])}@@ -427,14 +441,14 @@ func TestIndexChanged(t *testing.T) {r.opt.ChangedFrom = []string{filepath.Join(t.TempDir(), "empty.txt")}require.NoError(t, os.WriteFile(r.opt.ChangedFrom[0], nil, 0600))transfers, _ = r.index(t)- assert.Equal(t, int64(0), transfers)+ r.assertCount(t, 0, transfers)assert.NotContains(t, r.read(t, "sub/index.html"), "file6.txt")// Naming the root does a full runr.opt.ChangedFrom = nilr.opt.Changed = []string{"/"}transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/index.html"), "file6.txt")// So does exceeding --changed-max-dirs@@ -442,7 +456,7 @@ func TestIndexChanged(t *testing.T) {r.opt.Changed = []string{"other/notes.md"}r.opt.ChangedMaxDirs = 1transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/index.html"), "file7.txt")r.opt.ChangedMaxDirs = 0@@ -450,7 +464,7 @@ func TestIndexChanged(t *testing.T) {r.WriteObject(r.ctx, "sub/deep/file8.txt", "hello8", t1)r.opt.Changed = []string{"./sub/", "sub//deep/", "sub/deep/"}transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/deep/index.html"), "file8.txt")// A partial run doesn't write listings a full run wouldn't@@ -475,20 +489,20 @@ func TestIndexChangedDelete(t *testing.T) {r.opt.Changed = []string{"sub/deep/file3.txt"}transfers, deletes := r.index(t)if r.Fremote.Features().CanHaveEmptyDirectories {- assert.Equal(t, int64(1), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 1, transfers)+ r.assertCount(t, 0, deletes)assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file3.txt")} else {- assert.Equal(t, int64(1), transfers)- assert.Equal(t, int64(1), deletes)+ r.assertCount(t, 1, transfers)+ r.assertCount(t, 1, deletes)assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html")// Declaring a directory which has gone is harmlessr.opt.Changed = []string{"sub/deep/"}transfers, deletes = r.index(t)- assert.Equal(t, int64(0), transfers)- assert.Equal(t, int64(0), deletes)+ r.assertCount(t, 0, transfers)+ r.assertCount(t, 0, deletes)}}@@ -505,7 +519,7 @@ func TestIndexChangedFiles(t *testing.T) {require.NoError(t, os.WriteFile(from, []byte("- odd/file8.txt\n\n"), 0600))r.opt.ChangedFrom = []string{from}transfers, _ := r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "- odd/index.html"), "file8.txt")assert.NotContains(t, r.read(t, "sub/index.html"), "file9.txt")@@ -515,7 +529,7 @@ func TestIndexChangedFiles(t *testing.T) {r.opt.ChangedFrom = nilr.opt.ChangedCombined = []string{combined}transfers, _ = r.index(t)- assert.Equal(t, int64(1), transfers)+ r.assertCount(t, 1, transfers)assert.Contains(t, r.read(t, "sub/index.html"), "file9.txt")// Malformed combined lines are an error@@ -537,7 +551,11 @@ func TestIndexChangedDirTime(t *testing.T) {r.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", fstest.Time("2030-01-01T00:00:00Z"))r.opt.Changed = []string{"sub/deep/file4.txt"}transfers, _ := r.index(t)- assert.Equal(t, int64(3), transfers)+ if precision != fs.ModTimeNotSupported {+ // Without settable modification times the new file may not+ // change the time of the directories above it+ r.assertCount(t, 3, transfers)+ }modTimes := r.indexModTimes(t, "index.json")fstest.AssertTimeEqualWithPrecision(t, "sub", fstest.Time("2030-01-01T00:00:00Z"), modTimes["sub"], precision)fstest.AssertTimeEqualWithPrecision(t, "other", t2, modTimes["other"], precision)