Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fstest: add integration tests objects with // on bucket based backends #5858

Nick Craig-Wood81ecfb0f647c
diff --git a/fs/features.go b/fs/features.go
index 828f3b94b..cb9b69649 100644
--- a/fs/features.go
+++ b/fs/features.go
@@ -39,6 +39,7 @@ type Features struct {
NoMultiThreading bool // set if can't have multiplethreads on one download open
Overlay bool // this wraps one or more backends to add functionality
ChunkWriterDoesntSeek bool // set if the chunk writer doesn't need to read the data more than once
+ DoubleSlash bool // set if backend supports double slashes in paths
// Purge all files in the directory specified
//
@@ -383,6 +384,8 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
ft.PartialUploads = ft.PartialUploads && mask.PartialUploads
ft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading
// ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay
+ ft.ChunkWriterDoesntSeek = ft.ChunkWriterDoesntSeek && mask.ChunkWriterDoesntSeek
+ ft.DoubleSlash = ft.DoubleSlash && mask.DoubleSlash
if mask.Purge == nil {
ft.Purge = nil
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 2369e7c2d..c6903bc9a 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -2121,6 +2121,144 @@ func Run(t *testing.T, opt *Opt) {
}
})
+ // Run tests for bucket based Fs
+ // TestIntegration/FsMkdir/FsPutFiles/Bucket
+ t.Run("Bucket", func(t *testing.T) {
+ // Test if this Fs is bucket based - this test won't work for wrapped bucket based backends.
+ if !f.Features().BucketBased {
+ t.Skip("Not a bucket based backend")
+ }
+ if f.Features().CanHaveEmptyDirectories {
+ t.Skip("Can have empty directories")
+ }
+ if !f.Features().DoubleSlash {
+ t.Skip("Can't have // in paths")
+ }
+ // Create some troublesome file names
+ fileNames := []string{
+ file1.Path,
+ file2.Path,
+ ".leadingdot",
+ "/.leadingdot",
+ "///tripleslash",
+ "//doubleslash",
+ "dir/.leadingdot",
+ "dir///tripleslash",
+ "dir//doubleslash",
+ }
+ dirNames := []string{
+ "hello? sausage",
+ "hello? sausage/êé",
+ "hello? sausage/êé/Hello, 世界",
+ "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
+ "/",
+ "//",
+ "///",
+ "dir",
+ "dir/",
+ "dir//",
+ }
+ t1 := fstest.Time("2003-02-03T04:05:06.499999999Z")
+ var objs []fs.Object
+ for _, fileName := range fileNames[2:] {
+ contents := "bad file name: " + fileName
+ file := fstest.NewItem(fileName, contents, t1)
+ objs = append(objs, PutTestContents(ctx, t, f, &file, contents, true))
+ }
+
+ // Check they arrived
+ // This uses walk.Walk with a max size set to make sure we don't use ListR
+ check := func(f fs.Fs, dir string, wantFileNames, wantDirNames []string) {
+ t.Helper()
+ var gotFileNames, gotDirNames []string
+ require.NoError(t, walk.Walk(ctx, f, dir, true, 100, func(path string, entries fs.DirEntries, err error) error {
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ if _, isObj := entry.(fs.Object); isObj {
+ gotFileNames = append(gotFileNames, entry.Remote())
+ } else {
+ gotDirNames = append(gotDirNames, entry.Remote())
+ }
+ }
+ return nil
+ }))
+ sort.Strings(wantDirNames)
+ sort.Strings(wantFileNames)
+ sort.Strings(gotDirNames)
+ sort.Strings(gotFileNames)
+ assert.Equal(t, wantFileNames, gotFileNames)
+ assert.Equal(t, wantDirNames, gotDirNames)
+ }
+ check(f, "", fileNames, dirNames)
+ check(f, "/", []string{
+ "/.leadingdot",
+ "///tripleslash",
+ "//doubleslash",
+ }, []string{
+ "//",
+ "///",
+ })
+ check(f, "//", []string{
+ "///tripleslash",
+ "//doubleslash",
+ }, []string{
+ "///",
+ })
+ check(f, "dir", []string{
+ "dir/.leadingdot",
+ "dir///tripleslash",
+ "dir//doubleslash",
+ }, []string{
+ "dir/",
+ "dir//",
+ })
+ check(f, "dir/", []string{
+ "dir///tripleslash",
+ "dir//doubleslash",
+ }, []string{
+ "dir//",
+ })
+ check(f, "dir//", []string{
+ "dir///tripleslash",
+ }, nil)
+
+ // Now create a backend not at the root of a bucket
+ f2, err := fs.NewFs(ctx, subRemoteName+"/dir")
+ require.NoError(t, err)
+ check(f2, "", []string{
+ ".leadingdot",
+ "//tripleslash",
+ "/doubleslash",
+ }, []string{
+ "/",
+ "//",
+ })
+ check(f2, "/", []string{
+ "//tripleslash",
+ "/doubleslash",
+ }, []string{
+ "//",
+ })
+ check(f2, "//", []string{
+ "//tripleslash",
+ }, []string(nil))
+
+ // Remove the objects
+ for _, obj := range objs {
+ assert.NoError(t, obj.Remove(ctx))
+ }
+
+ // Check they are gone
+ fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{
+ "hello? sausage",
+ "hello? sausage/êé",
+ "hello? sausage/êé/Hello, 世界",
+ "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
+ }, fs.GetModifyWindow(ctx, f))
+ })
+
// State of remote at the moment the internal tests are called
InternalTestFiles = []fstest.Item{file1, file2}