"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fstest: add integration tests objects with // on bucket based backends #5858
Nick Craig-Wood81ecfb0f647c
diff --git a/fs/features.go b/fs/features.goindex 828f3b94b..cb9b69649 100644--- a/fs/features.go+++ b/fs/features.go@@ -39,6 +39,7 @@ type Features struct {NoMultiThreading bool // set if can't have multiplethreads on one download openOverlay bool // this wraps one or more backends to add functionalityChunkWriterDoesntSeek bool // set if the chunk writer doesn't need to read the data more than once+ DoubleSlash bool // set if backend supports double slashes in paths// Purge all files in the directory specified//@@ -383,6 +384,8 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {ft.PartialUploads = ft.PartialUploads && mask.PartialUploadsft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading// ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay+ ft.ChunkWriterDoesntSeek = ft.ChunkWriterDoesntSeek && mask.ChunkWriterDoesntSeek+ ft.DoubleSlash = ft.DoubleSlash && mask.DoubleSlashif mask.Purge == nil {ft.Purge = nildiff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.goindex 2369e7c2d..c6903bc9a 100644--- a/fstest/fstests/fstests.go+++ b/fstest/fstests/fstests.go@@ -2121,6 +2121,144 @@ func Run(t *testing.T, opt *Opt) {}})+ // Run tests for bucket based Fs+ // TestIntegration/FsMkdir/FsPutFiles/Bucket+ t.Run("Bucket", func(t *testing.T) {+ // Test if this Fs is bucket based - this test won't work for wrapped bucket based backends.+ if !f.Features().BucketBased {+ t.Skip("Not a bucket based backend")+ }+ if f.Features().CanHaveEmptyDirectories {+ t.Skip("Can have empty directories")+ }+ if !f.Features().DoubleSlash {+ t.Skip("Can't have // in paths")+ }+ // Create some troublesome file names+ fileNames := []string{+ file1.Path,+ file2.Path,+ ".leadingdot",+ "/.leadingdot",+ "///tripleslash",+ "//doubleslash",+ "dir/.leadingdot",+ "dir///tripleslash",+ "dir//doubleslash",+ }+ dirNames := []string{+ "hello? sausage",+ "hello? sausage/êé",+ "hello? sausage/êé/Hello, 世界",+ "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",+ "/",+ "//",+ "///",+ "dir",+ "dir/",+ "dir//",+ }+ t1 := fstest.Time("2003-02-03T04:05:06.499999999Z")+ var objs []fs.Object+ for _, fileName := range fileNames[2:] {+ contents := "bad file name: " + fileName+ file := fstest.NewItem(fileName, contents, t1)+ objs = append(objs, PutTestContents(ctx, t, f, &file, contents, true))+ }++ // Check they arrived+ // This uses walk.Walk with a max size set to make sure we don't use ListR+ check := func(f fs.Fs, dir string, wantFileNames, wantDirNames []string) {+ t.Helper()+ var gotFileNames, gotDirNames []string+ require.NoError(t, walk.Walk(ctx, f, dir, true, 100, func(path string, entries fs.DirEntries, err error) error {+ if err != nil {+ return err+ }+ for _, entry := range entries {+ if _, isObj := entry.(fs.Object); isObj {+ gotFileNames = append(gotFileNames, entry.Remote())+ } else {+ gotDirNames = append(gotDirNames, entry.Remote())+ }+ }+ return nil+ }))+ sort.Strings(wantDirNames)+ sort.Strings(wantFileNames)+ sort.Strings(gotDirNames)+ sort.Strings(gotFileNames)+ assert.Equal(t, wantFileNames, gotFileNames)+ assert.Equal(t, wantDirNames, gotDirNames)+ }+ check(f, "", fileNames, dirNames)+ check(f, "/", []string{+ "/.leadingdot",+ "///tripleslash",+ "//doubleslash",+ }, []string{+ "//",+ "///",+ })+ check(f, "//", []string{+ "///tripleslash",+ "//doubleslash",+ }, []string{+ "///",+ })+ check(f, "dir", []string{+ "dir/.leadingdot",+ "dir///tripleslash",+ "dir//doubleslash",+ }, []string{+ "dir/",+ "dir//",+ })+ check(f, "dir/", []string{+ "dir///tripleslash",+ "dir//doubleslash",+ }, []string{+ "dir//",+ })+ check(f, "dir//", []string{+ "dir///tripleslash",+ }, nil)++ // Now create a backend not at the root of a bucket+ f2, err := fs.NewFs(ctx, subRemoteName+"/dir")+ require.NoError(t, err)+ check(f2, "", []string{+ ".leadingdot",+ "//tripleslash",+ "/doubleslash",+ }, []string{+ "/",+ "//",+ })+ check(f2, "/", []string{+ "//tripleslash",+ "/doubleslash",+ }, []string{+ "//",+ })+ check(f2, "//", []string{+ "//tripleslash",+ }, []string(nil))++ // Remove the objects+ for _, obj := range objs {+ assert.NoError(t, obj.Remove(ctx))+ }++ // Check they are gone+ fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{+ "hello? sausage",+ "hello? sausage/êé",+ "hello? sausage/êé/Hello, 世界",+ "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",+ }, fs.GetModifyWindow(ctx, f))+ })+// State of remote at the moment the internal tests are calledInternalTestFiles = []fstest.Item{file1, file2}