Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fstests: add integration tests for Directory Metadata and ModTime

Nick Craig-Wood61d76ae47dbe
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 57cf76d27..6f55b09c0 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1639,3 +1639,112 @@ func TestTouchDir(t *testing.T) {
r.CheckRemoteItems(t, file1, file2, file3)
}
}
+
+var testMetadata = fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": t1.Format(time.RFC3339Nano),
+ // User metadata
+ "potato": "jersey",
+}
+
+func TestMkdirMetadata(t *testing.T) {
+ const name = "dir with metadata"
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ features := r.Fremote.Features()
+ if features.MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support MkdirMetadata")
+ }
+
+ newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata)
+}
+
+func TestMkdirModTime(t *testing.T) {
+ const name = "directory with modtime"
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")
+ }
+ newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)
+ require.NoError(t, err)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+}
+
+func TestCopyDirMetadata(t *testing.T) {
+ const nameNonExistent = "non existent directory"
+ const nameExistent = "existing directory"
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
+ }
+
+ // Create a source local directory with metadata
+ newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, newSrc)
+
+ // First try with the directory not existing
+ newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata)
+
+ // Then try with the directory existing
+ require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent))
+ require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent))
+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent)
+
+ newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata)
+}
+
+func TestSetDirModTime(t *testing.T) {
+ const name = "set modtime on existing directory"
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime {
+ t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime")
+ }
+
+ // First try with the directory not existing - should return an error
+ newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
+ require.Error(t, err)
+ require.Nil(t, newDst)
+
+ // Then try with the directory existing
+ require.NoError(t, r.Fremote.Mkdir(ctx, name))
+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
+
+ newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+}
diff --git a/fstest/fstest.go b/fstest/fstest.go
index c74ae6e9a..bff0b027e 100644
--- a/fstest/fstest.go
+++ b/fstest/fstest.go
@@ -519,3 +519,71 @@ func Purge(f fs.Fs) {
log.Printf("purge failed: %v", err)
}
}
+
+// NewDirectory finds the directory with remote in f
+//
+// One day this will be an rclone primitive
+func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory {
+ var err error
+ var dir fs.Directory
+ sleepTime := 1 * time.Second
+ root := path.Dir(remote)
+ if root == "." {
+ root = ""
+ }
+ for i := 1; i <= *ListRetries; i++ {
+ var entries fs.DirEntries
+ entries, err = f.List(ctx, root)
+ if err != nil {
+ continue
+ }
+ for _, entry := range entries {
+ var ok bool
+ dir, ok = entry.(fs.Directory)
+ if ok && dir.Remote() == remote {
+ return dir
+ }
+ }
+ err = fmt.Errorf("directory %q not found in %q", remote, root)
+ t.Logf("Sleeping for %v for findDir eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err)
+ time.Sleep(sleepTime)
+ sleepTime = (sleepTime * 3) / 2
+ }
+ require.NoError(t, err)
+ return dir
+}
+
+// CheckEntryMetadata checks the metadata on the directory
+//
+// This checks a limited set of metadata on the directory
+func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) {
+ features := f.Features()
+ do, ok := entry.(fs.Metadataer)
+ require.True(t, ok, "Didn't find expected Metadata() method on %T", entry)
+ gotMeta, err := do.Metadata(ctx)
+ require.NoError(t, err)
+
+ for k, v := range wantMeta {
+ switch k {
+ case "mtime", "atime", "btime", "ctime":
+ // Check the system time Metadata
+ wantT, err := time.Parse(time.RFC3339, v)
+ require.NoError(t, err)
+ gotT, err := time.Parse(time.RFC3339, gotMeta[k])
+ require.NoError(t, err)
+ AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision())
+ default:
+ // Check the User metadata if we can
+ _, isDir := entry.(fs.Directory)
+ if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) {
+ assert.Equal(t, v, gotMeta[k])
+ }
+ }
+ }
+}
+
+// CheckDirModTime checks the modtime on the directory
+func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) {
+ gotT := dir.ModTime(ctx)
+ AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision())
+}
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 095a94d23..e978e159d 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -307,19 +307,21 @@ type ExtraConfigItem struct{ Name, Key, Value string }
// Opt is options for Run
type Opt struct {
- RemoteName string
- NilObject fs.Object
- ExtraConfig []ExtraConfigItem
- SkipBadWindowsCharacters bool // skips unusable characters for windows if set
- SkipFsMatch bool // if set skip exact matching of Fs value
- TiersToTest []string // List of tiers which can be tested in setTier test
- ChunkedUpload ChunkedUploadConfig
- UnimplementableFsMethods []string // List of methods which can't be implemented in this wrapping Fs
- UnimplementableObjectMethods []string // List of methods which can't be implemented in this wrapping Fs
- SkipFsCheckWrap bool // if set skip FsCheckWrap
- SkipObjectCheckWrap bool // if set skip ObjectCheckWrap
- SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks
- QuickTestOK bool // if set, run this test with make quicktest
+ RemoteName string
+ NilObject fs.Object
+ ExtraConfig []ExtraConfigItem
+ SkipBadWindowsCharacters bool // skips unusable characters for windows if set
+ SkipFsMatch bool // if set skip exact matching of Fs value
+ TiersToTest []string // List of tiers which can be tested in setTier test
+ ChunkedUpload ChunkedUploadConfig
+ UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs
+ UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs
+ UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs
+ SkipFsCheckWrap bool // if set skip FsCheckWrap
+ SkipObjectCheckWrap bool // if set skip ObjectCheckWrap
+ SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap
+ SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks
+ QuickTestOK bool // if set, run this test with make quicktest
}
// returns true if x is found in ss
@@ -1513,8 +1515,8 @@ func Run(t *testing.T, opt *Opt) {
}
}
if !features.ReadMetadata {
- if metadata != nil {
- require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata")
+ if metadata != nil && !features.Overlay {
+ require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata)
}
} else if features.WriteMetadata {
require.NotNil(t, metadata)
@@ -2301,6 +2303,190 @@ func Run(t *testing.T, opt *Opt) {
// somehow confused about root and absolute root.
})
+ // FsDirSetModTime tests setting the mod time on a directory if possible
+ t.Run("FsDirSetModTime", func(t *testing.T) {
+ const name = "dir-mod-time"
+ do := f.Features().DirSetModTime
+ if do == nil {
+ t.Skip("FS has no DirSetModTime interface")
+ }
+
+ // Set ModTime on non existing directory should return error
+ t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+ err := do(ctx, name, t1)
+ require.Error(t, err)
+
+ // Make the directory and try again
+ err = f.Mkdir(ctx, name)
+ require.NoError(t, err)
+ err = do(ctx, name, t1)
+ require.NoError(t, err)
+
+ // Check the modtime got set properly
+ dir := fstest.NewDirectory(ctx, t, f, name)
+ fstest.CheckDirModTime(ctx, t, f, dir, t1)
+
+ // Tidy up
+ err = f.Rmdir(ctx, name)
+ require.NoError(t, err)
+ })
+
+ var testMetadata = fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": "2001-02-03T04:05:06.499999999Z",
+ // User metadata
+ "potato": "jersey",
+ }
+ var testMetadata2 = fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": "2002-02-03T04:05:06.499999999Z",
+ // User metadata
+ "potato": "king edwards",
+ }
+
+ // FsMkdirMetadata tests creating a directory with metadata if possible
+ t.Run("FsMkdirMetadata", func(t *testing.T) {
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ const name = "dir-metadata"
+ do := f.Features().MkdirMetadata
+ if do == nil {
+ t.Skip("FS has no MkdirMetadata interface")
+ }
+ assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
+
+ // Create the directory from fresh
+ dir, err := do(ctx, name, testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, dir)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
+
+ // Now update the metadata on the existing directory
+ t.Run("Update", func(t *testing.T) {
+ dir, err := do(ctx, name, testMetadata2)
+ require.NoError(t, err)
+ require.NotNil(t, dir)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2)
+ // The TestUnionPolicy2 has randomness in it so it sets metadata on
+ // one directory but can read a different one from the listing.
+ if f.Name() != "TestUnionPolicy2" {
+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2)
+ }
+ })
+
+ // Now test the Directory methods
+ t.Run("CheckDirectory", func(t *testing.T) {
+ _, ok := dir.(fs.Object)
+ assert.False(t, ok, "Directory must not type assert to Object")
+ _, ok = dir.(fs.ObjectInfo)
+ assert.False(t, ok, "Directory must not type assert to ObjectInfo")
+ })
+
+ // Tidy up
+ err = f.Rmdir(ctx, name)
+ require.NoError(t, err)
+ })
+
+ // FsDirectory checks methods on the directory object
+ t.Run("FsDirectory", func(t *testing.T) {
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ const name = "dir-methods"
+ features := f.Features()
+
+ if !features.CanHaveEmptyDirectories {
+ t.Skip("Can't test if can't have empty directories")
+ }
+ if !features.ReadDirMetadata &&
+ !features.WriteDirMetadata &&
+ !features.WriteDirSetModTime &&
+ !features.UserDirMetadata &&
+ !features.Overlay &&
+ features.UnWrap == nil {
+ t.Skip("FS has no Directory methods and doesn't Wrap")
+ }
+
+ // Create a directory to start with
+ err := f.Mkdir(ctx, name)
+ require.NoError(t, err)
+
+ // Get the directory object
+ dir := fstest.NewDirectory(ctx, t, f, name)
+ _, ok := dir.(fs.Object)
+ assert.False(t, ok, "Directory must not type assert to Object")
+ _, ok = dir.(fs.ObjectInfo)
+ assert.False(t, ok, "Directory must not type assert to ObjectInfo")
+
+ // Now test the directory methods
+ t.Run("ReadDirMetadata", func(t *testing.T) {
+ if !features.ReadDirMetadata {
+ t.Skip("Directories don't support ReadDirMetadata")
+ }
+ if f.Name() == "TestUnionPolicy3" {
+ t.Skipf("Test unreliable on %q", f.Name())
+ }
+ fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{
+ "mtime": dir.ModTime(ctx).Format(time.RFC3339Nano),
+ })
+ })
+
+ t.Run("WriteDirMetadata", func(t *testing.T) {
+ if !features.WriteDirMetadata {
+ t.Skip("Directories don't support WriteDirMetadata")
+ }
+ assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
+ do, ok := dir.(fs.SetMetadataer)
+ require.True(t, ok, "Expected to find SetMetadata method on Directory")
+ err := do.SetMetadata(ctx, testMetadata)
+ require.NoError(t, err)
+
+ fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
+ })
+
+ t.Run("WriteDirSetModTime", func(t *testing.T) {
+ if !features.WriteDirSetModTime {
+ t.Skip("Directories don't support WriteDirSetModTime")
+ }
+ assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime")
+
+ t1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
+
+ do, ok := dir.(fs.SetModTimer)
+ require.True(t, ok, "Expected to find SetMetadata method on Directory")
+ err := do.SetModTime(ctx, t1)
+ require.NoError(t, err)
+
+ fstest.CheckDirModTime(ctx, t, f, dir, t1)
+ fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1)
+ })
+
+ // Check to see if Fs that wrap other Directories implement all the optional methods
+ t.Run("DirectoryCheckWrap", func(t *testing.T) {
+ if opt.SkipDirectoryCheckWrap {
+ t.Skip("Skipping DirectoryCheckWrap on this Fs")
+ }
+ if !features.Overlay && features.UnWrap == nil {
+ t.Skip("Not a wrapping Fs")
+ }
+ _, unsupported := fs.DirectoryOptionalInterfaces(dir)
+ for _, name := range unsupported {
+ if !stringsContains(name, opt.UnimplementableDirectoryMethods) {
+ t.Errorf("Missing Directory wrapper for %s", name)
+ }
+ }
+ })
+
+ // Tidy up
+ err = f.Rmdir(ctx, name)
+ require.NoError(t, err)
+ })
+
// Purge the folder
err = operations.Purge(ctx, f, "")
if !errors.Is(err, fs.ErrorDirNotFound) {