"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fstests: add integration tests for Directory Metadata and ModTime
Nick Craig-Wood61d76ae47dbe
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex 57cf76d27..6f55b09c0 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -1639,3 +1639,112 @@ func TestTouchDir(t *testing.T) {r.CheckRemoteItems(t, file1, file2, file3)}}++var testMetadata = fs.Metadata{+ // System metadata supported by all backends+ "mtime": t1.Format(time.RFC3339Nano),+ // User metadata+ "potato": "jersey",+}++func TestMkdirMetadata(t *testing.T) {+ const name = "dir with metadata"+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ ci.Metadata = true+ r := fstest.NewRun(t)+ features := r.Fremote.Features()+ if features.MkdirMetadata == nil {+ t.Skip("Skipping test as remote does not support MkdirMetadata")+ }++ newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)+ require.NoError(t, err)+ require.NotNil(t, newDst)++ require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")++ // Check the returned directory and one read from the listing+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata)+}++func TestMkdirModTime(t *testing.T) {+ const name = "directory with modtime"+ ctx := context.Background()+ r := fstest.NewRun(t)+ if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {+ t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")+ }+ newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)+ require.NoError(t, err)++ // Check the returned directory and one read from the listing+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)+}++func TestCopyDirMetadata(t *testing.T) {+ const nameNonExistent = "non existent directory"+ const nameExistent = "existing directory"+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ ci.Metadata = true+ r := fstest.NewRun(t)+ if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {+ t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")+ }++ // Create a source local directory with metadata+ newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)+ require.NoError(t, err)+ require.NotNil(t, newSrc)++ // First try with the directory not existing+ newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)+ require.NoError(t, err)+ require.NotNil(t, newDst)++ // Check the returned directory and one read from the listing+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata)++ // Then try with the directory existing+ require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent))+ require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent))+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent)++ newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc)+ require.NoError(t, err)+ require.NotNil(t, newDst)++ // Check the returned directory and one read from the listing+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata)+}++func TestSetDirModTime(t *testing.T) {+ const name = "set modtime on existing directory"+ ctx := context.Background()+ r := fstest.NewRun(t)+ if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime {+ t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime")+ }++ // First try with the directory not existing - should return an error+ newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)+ require.Error(t, err)+ require.Nil(t, newDst)++ // Then try with the directory existing+ require.NoError(t, r.Fremote.Mkdir(ctx, name))+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)++ newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)+ require.NoError(t, err)+ require.NotNil(t, newDst)++ // Check the returned directory and one read from the listing+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)+}diff --git a/fstest/fstest.go b/fstest/fstest.goindex c74ae6e9a..bff0b027e 100644--- a/fstest/fstest.go+++ b/fstest/fstest.go@@ -519,3 +519,71 @@ func Purge(f fs.Fs) {log.Printf("purge failed: %v", err)}}++// NewDirectory finds the directory with remote in f+//+// One day this will be an rclone primitive+func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory {+ var err error+ var dir fs.Directory+ sleepTime := 1 * time.Second+ root := path.Dir(remote)+ if root == "." {+ root = ""+ }+ for i := 1; i <= *ListRetries; i++ {+ var entries fs.DirEntries+ entries, err = f.List(ctx, root)+ if err != nil {+ continue+ }+ for _, entry := range entries {+ var ok bool+ dir, ok = entry.(fs.Directory)+ if ok && dir.Remote() == remote {+ return dir+ }+ }+ err = fmt.Errorf("directory %q not found in %q", remote, root)+ t.Logf("Sleeping for %v for findDir eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err)+ time.Sleep(sleepTime)+ sleepTime = (sleepTime * 3) / 2+ }+ require.NoError(t, err)+ return dir+}++// CheckEntryMetadata checks the metadata on the directory+//+// This checks a limited set of metadata on the directory+func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) {+ features := f.Features()+ do, ok := entry.(fs.Metadataer)+ require.True(t, ok, "Didn't find expected Metadata() method on %T", entry)+ gotMeta, err := do.Metadata(ctx)+ require.NoError(t, err)++ for k, v := range wantMeta {+ switch k {+ case "mtime", "atime", "btime", "ctime":+ // Check the system time Metadata+ wantT, err := time.Parse(time.RFC3339, v)+ require.NoError(t, err)+ gotT, err := time.Parse(time.RFC3339, gotMeta[k])+ require.NoError(t, err)+ AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision())+ default:+ // Check the User metadata if we can+ _, isDir := entry.(fs.Directory)+ if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) {+ assert.Equal(t, v, gotMeta[k])+ }+ }+ }+}++// CheckDirModTime checks the modtime on the directory+func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) {+ gotT := dir.ModTime(ctx)+ AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision())+}diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.goindex 095a94d23..e978e159d 100644--- a/fstest/fstests/fstests.go+++ b/fstest/fstests/fstests.go@@ -307,19 +307,21 @@ type ExtraConfigItem struct{ Name, Key, Value string }// Opt is options for Runtype Opt struct {- RemoteName string- NilObject fs.Object- ExtraConfig []ExtraConfigItem- SkipBadWindowsCharacters bool // skips unusable characters for windows if set- SkipFsMatch bool // if set skip exact matching of Fs value- TiersToTest []string // List of tiers which can be tested in setTier test- ChunkedUpload ChunkedUploadConfig- UnimplementableFsMethods []string // List of methods which can't be implemented in this wrapping Fs- UnimplementableObjectMethods []string // List of methods which can't be implemented in this wrapping Fs- SkipFsCheckWrap bool // if set skip FsCheckWrap- SkipObjectCheckWrap bool // if set skip ObjectCheckWrap- SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks- QuickTestOK bool // if set, run this test with make quicktest+ RemoteName string+ NilObject fs.Object+ ExtraConfig []ExtraConfigItem+ SkipBadWindowsCharacters bool // skips unusable characters for windows if set+ SkipFsMatch bool // if set skip exact matching of Fs value+ TiersToTest []string // List of tiers which can be tested in setTier test+ ChunkedUpload ChunkedUploadConfig+ UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs+ UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs+ UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs+ SkipFsCheckWrap bool // if set skip FsCheckWrap+ SkipObjectCheckWrap bool // if set skip ObjectCheckWrap+ SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap+ SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks+ QuickTestOK bool // if set, run this test with make quicktest}// returns true if x is found in ss@@ -1513,8 +1515,8 @@ func Run(t *testing.T, opt *Opt) {}}if !features.ReadMetadata {- if metadata != nil {- require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata")+ if metadata != nil && !features.Overlay {+ require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata)}} else if features.WriteMetadata {require.NotNil(t, metadata)@@ -2301,6 +2303,190 @@ func Run(t *testing.T, opt *Opt) {// somehow confused about root and absolute root.})+ // FsDirSetModTime tests setting the mod time on a directory if possible+ t.Run("FsDirSetModTime", func(t *testing.T) {+ const name = "dir-mod-time"+ do := f.Features().DirSetModTime+ if do == nil {+ t.Skip("FS has no DirSetModTime interface")+ }++ // Set ModTime on non existing directory should return error+ t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")+ err := do(ctx, name, t1)+ require.Error(t, err)++ // Make the directory and try again+ err = f.Mkdir(ctx, name)+ require.NoError(t, err)+ err = do(ctx, name, t1)+ require.NoError(t, err)++ // Check the modtime got set properly+ dir := fstest.NewDirectory(ctx, t, f, name)+ fstest.CheckDirModTime(ctx, t, f, dir, t1)++ // Tidy up+ err = f.Rmdir(ctx, name)+ require.NoError(t, err)+ })++ var testMetadata = fs.Metadata{+ // System metadata supported by all backends+ "mtime": "2001-02-03T04:05:06.499999999Z",+ // User metadata+ "potato": "jersey",+ }+ var testMetadata2 = fs.Metadata{+ // System metadata supported by all backends+ "mtime": "2002-02-03T04:05:06.499999999Z",+ // User metadata+ "potato": "king edwards",+ }++ // FsMkdirMetadata tests creating a directory with metadata if possible+ t.Run("FsMkdirMetadata", func(t *testing.T) {+ ctx, ci := fs.AddConfig(ctx)+ ci.Metadata = true+ const name = "dir-metadata"+ do := f.Features().MkdirMetadata+ if do == nil {+ t.Skip("FS has no MkdirMetadata interface")+ }+ assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")++ // Create the directory from fresh+ dir, err := do(ctx, name, testMetadata)+ require.NoError(t, err)+ require.NotNil(t, dir)++ // Check the returned directory and one read from the listing+ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)++ // Now update the metadata on the existing directory+ t.Run("Update", func(t *testing.T) {+ dir, err := do(ctx, name, testMetadata2)+ require.NoError(t, err)+ require.NotNil(t, dir)++ // Check the returned directory and one read from the listing+ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2)+ // The TestUnionPolicy2 has randomness in it so it sets metadata on+ // one directory but can read a different one from the listing.+ if f.Name() != "TestUnionPolicy2" {+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2)+ }+ })++ // Now test the Directory methods+ t.Run("CheckDirectory", func(t *testing.T) {+ _, ok := dir.(fs.Object)+ assert.False(t, ok, "Directory must not type assert to Object")+ _, ok = dir.(fs.ObjectInfo)+ assert.False(t, ok, "Directory must not type assert to ObjectInfo")+ })++ // Tidy up+ err = f.Rmdir(ctx, name)+ require.NoError(t, err)+ })++ // FsDirectory checks methods on the directory object+ t.Run("FsDirectory", func(t *testing.T) {+ ctx, ci := fs.AddConfig(ctx)+ ci.Metadata = true+ const name = "dir-methods"+ features := f.Features()++ if !features.CanHaveEmptyDirectories {+ t.Skip("Can't test if can't have empty directories")+ }+ if !features.ReadDirMetadata &&+ !features.WriteDirMetadata &&+ !features.WriteDirSetModTime &&+ !features.UserDirMetadata &&+ !features.Overlay &&+ features.UnWrap == nil {+ t.Skip("FS has no Directory methods and doesn't Wrap")+ }++ // Create a directory to start with+ err := f.Mkdir(ctx, name)+ require.NoError(t, err)++ // Get the directory object+ dir := fstest.NewDirectory(ctx, t, f, name)+ _, ok := dir.(fs.Object)+ assert.False(t, ok, "Directory must not type assert to Object")+ _, ok = dir.(fs.ObjectInfo)+ assert.False(t, ok, "Directory must not type assert to ObjectInfo")++ // Now test the directory methods+ t.Run("ReadDirMetadata", func(t *testing.T) {+ if !features.ReadDirMetadata {+ t.Skip("Directories don't support ReadDirMetadata")+ }+ if f.Name() == "TestUnionPolicy3" {+ t.Skipf("Test unreliable on %q", f.Name())+ }+ fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{+ "mtime": dir.ModTime(ctx).Format(time.RFC3339Nano),+ })+ })++ t.Run("WriteDirMetadata", func(t *testing.T) {+ if !features.WriteDirMetadata {+ t.Skip("Directories don't support WriteDirMetadata")+ }+ assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")+ do, ok := dir.(fs.SetMetadataer)+ require.True(t, ok, "Expected to find SetMetadata method on Directory")+ err := do.SetMetadata(ctx, testMetadata)+ require.NoError(t, err)++ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)+ })++ t.Run("WriteDirSetModTime", func(t *testing.T) {+ if !features.WriteDirSetModTime {+ t.Skip("Directories don't support WriteDirSetModTime")+ }+ assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime")++ t1 := fstest.Time("2001-02-03T04:05:10.123123123Z")++ do, ok := dir.(fs.SetModTimer)+ require.True(t, ok, "Expected to find SetMetadata method on Directory")+ err := do.SetModTime(ctx, t1)+ require.NoError(t, err)++ fstest.CheckDirModTime(ctx, t, f, dir, t1)+ fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1)+ })++ // Check to see if Fs that wrap other Directories implement all the optional methods+ t.Run("DirectoryCheckWrap", func(t *testing.T) {+ if opt.SkipDirectoryCheckWrap {+ t.Skip("Skipping DirectoryCheckWrap on this Fs")+ }+ if !features.Overlay && features.UnWrap == nil {+ t.Skip("Not a wrapping Fs")+ }+ _, unsupported := fs.DirectoryOptionalInterfaces(dir)+ for _, name := range unsupported {+ if !stringsContains(name, opt.UnimplementableDirectoryMethods) {+ t.Errorf("Missing Directory wrapper for %s", name)+ }+ }+ })++ // Tidy up+ err = f.Rmdir(ctx, name)+ require.NoError(t, err)+ })+// Purge the foldererr = operations.Purge(ctx, f, "")if !errors.Is(err, fs.ErrorDirNotFound) {