"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfstest: make VFS test suite support symlinks
Filipe Azevedof1d2f2b2c855
diff --git a/vfs/vfstest/file.go b/vfs/vfstest/file.goindex a4e490f0e..e923b521f 100644--- a/vfs/vfstest/file.go+++ b/vfs/vfstest/file.go@@ -6,6 +6,7 @@ import ("testing""time"+ "github.com/rclone/rclone/fs""github.com/rclone/rclone/vfs""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -72,3 +73,230 @@ func TestFileModTimeWithOpenWriters(t *testing.T) {run.rm(t, "cp-archive-test")}++// TestSymlinks tests all the api of the VFS / Mount symlinks support+func TestSymlinks(t *testing.T) {+ run.skipIfNoFUSE(t)+ if !run.vfsOpt.Links {+ t.Skip("No symlinks configured")+ }++ if runtime.GOOS == "windows" {+ t.Skip("Skipping test on Windows")+ }++ fs.Logf(nil, "Links: %v, useVFS: %v, suffix: %v", run.vfsOpt.Links, run.useVFS, fs.LinkSuffix)++ // Create initial setup of test files and directories we will create links to+ run.mkdir(t, "dir1")+ run.mkdir(t, "dir1/sub1dir1")+ run.createFile(t, "dir1/file1", "potato")+ run.mkdir(t, "dir2")+ run.mkdir(t, "dir2/sub1dir2")+ run.createFile(t, "dir2/file1", "chicken")++ // base state all the tests will be build off+ baseState := "dir1/|dir1/sub1dir1/|dir1/file1 6|dir2/|dir2/sub1dir2/|dir2/file1 7"+ // Check the tests return to the base state+ checkBaseState := func() {+ run.checkDir(t, baseState)+ }+ checkBaseState()++ t.Run("FileLink", func(t *testing.T) {+ // Link to a file+ run.symlink(t, "dir1/file1", "dir1file1_link")+ run.checkDir(t, baseState+"|dir1file1_link 10")+ run.checkMode(t, "dir1file1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.FilePerms))+ assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link"))++ // Read through a symlink+ assert.Equal(t, "potato", run.readFile(t, "dir1file1_link"))++ // Write through a symlink+ err := writeFile(run.path("dir1file1_link"), []byte("carrot"), 0600)+ require.NoError(t, err)+ assert.Equal(t, "carrot", run.readFile(t, "dir1file1_link"))+ assert.Equal(t, "carrot", run.readFile(t, "dir1/file1"))++ // Rename a symlink+ err = run.os.Rename(run.path("dir1file1_link"), run.path("dir1file1_link")+"_bla")+ require.NoError(t, err)+ run.checkDir(t, baseState+"|dir1file1_link_bla 10")+ assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link_bla"))++ // Delete a symlink+ run.rm(t, "dir1file1_link_bla")+ checkBaseState()+ })++ t.Run("DirLink", func(t *testing.T) {+ // Link to a dir+ run.symlink(t, "dir1", "dir1_link")+ run.checkDir(t, baseState+"|dir1_link 4")+ run.checkMode(t, "dir1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.DirPerms))+ assert.Equal(t, "dir1", run.readlink(t, "dir1_link"))++ // Check you can't open a directory symlink+ _, err := run.os.OpenFile(run.path("dir1_link"), os.O_WRONLY, 0600)+ require.Error(t, err)++ // Our symlink resolution is very simple when using the VFS as when using the+ // mount the OS will resolve the symlinks, so we don't recurse here++ // Read entries directly+ dir1Entries := make(dirMap)+ run.readLocalEx(t, dir1Entries, "dir1", false)+ assert.Equal(t, newDirMap("dir1/sub1dir1/|dir1/file1 6"), dir1Entries)++ // Read entries through the directory symlink+ dir1EntriesSymlink := make(dirMap)+ run.readLocalEx(t, dir1EntriesSymlink, "dir1_link", false)+ assert.Equal(t, newDirMap("dir1_link/sub1dir1/|dir1_link/file1 6"), dir1EntriesSymlink)++ // Rename directory symlink+ err = run.os.Rename(run.path("dir1_link"), run.path("dir1_link")+"_bla")+ require.NoError(t, err)+ run.checkDir(t, baseState+"|dir1_link_bla 4")+ assert.Equal(t, "dir1", run.readlink(t, "dir1_link_bla"))++ // Remove directory symlink+ run.rm(t, "dir1_link_bla")++ checkBaseState()+ })++ // Corner case #1 - We do not allow creating regular and symlink files having the same name (ie, test.txt and test.txt.rclonelink)++ // Symlink first, then regular+ t.Run("OverwriteSymlinkWithRegular", func(t *testing.T) {+ link1Name := "link1.txt"++ run.symlink(t, "dir1/file1", link1Name)+ run.checkDir(t, baseState+"|link1.txt 10")++ fh, err := run.os.OpenFile(run.path(link1Name), os.O_WRONLY|os.O_CREATE, os.FileMode(run.vfsOpt.FilePerms))++ // On real mount with links enabled, that open the symlink target as expected, else that fails to create a new file+ assert.NoError(t, err)+ // Don't care about the result, in some cache mode the file can't be opened for writing, so closing would trigger an err+ _ = fh.Close()++ run.rm(t, link1Name)+ checkBaseState()+ })++ // Regular first, then symlink+ t.Run("OverwriteRegularWithSymlink", func(t *testing.T) {+ link1Name := "link1.txt"++ run.createFile(t, link1Name, "")+ run.checkDir(t, baseState+"|link1.txt 0")++ err := run.os.Symlink(".", run.path(link1Name))+ assert.Error(t, err)++ run.rm(t, link1Name)+ checkBaseState()+ })++ // Corner case #2 - We do not allow creating directory and symlink file having the same name (ie, test and test.rclonelink)++ // Symlink first, then directory+ t.Run("OverwriteSymlinkWithDirectory", func(t *testing.T) {+ link1Name := "link1"++ run.symlink(t, ".", link1Name)+ run.checkDir(t, baseState+"|link1 1")++ err := run.os.Mkdir(run.path(link1Name), os.FileMode(run.vfsOpt.DirPerms))+ assert.Error(t, err)++ run.rm(t, link1Name)+ checkBaseState()+ })++ // Directory first, then symlink+ t.Run("OverwriteDirectoryWithSymlink", func(t *testing.T) {+ link1Name := "link1"++ run.mkdir(t, link1Name)+ run.checkDir(t, baseState+"|link1/")++ err := run.os.Symlink(".", run.path(link1Name))+ assert.Error(t, err)++ run.rm(t, link1Name)+ checkBaseState()+ })++ // Corner case #3 - We do not allow moving directory or file having the same name in a target (ie, test and test.rclonelink)++ // Move symlink -> regular file+ t.Run("MoveSymlinkToFile", func(t *testing.T) {+ t.Skip("FIXME not implemented")+ link1Name := "link1.txt"++ run.symlink(t, ".", link1Name)+ run.createFile(t, "dir1/link1.txt", "")+ run.checkDir(t, baseState+"|link1.txt 1|dir1/link1.txt 0")++ err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))+ assert.Error(t, err)++ run.rm(t, link1Name)+ run.rm(t, "dir1/link1.txt")+ checkBaseState()+ })++ // Move regular file -> symlink+ t.Run("MoveFileToSymlink", func(t *testing.T) {+ t.Skip("FIXME not implemented")+ link1Name := "link1.txt"++ run.createFile(t, link1Name, "")+ run.symlink(t, ".", "dir1/"+link1Name)+ run.checkDir(t, baseState+"|link1.txt 0|dir1/link1.txt 1")++ err := run.os.Rename(run.path(link1Name), run.path("dir1/link1.txt"))+ assert.Error(t, err)++ run.rm(t, link1Name)+ run.rm(t, "dir1/"+link1Name)+ checkBaseState()+ })++ // Move symlink -> directory+ t.Run("MoveSymlinkToDirectory", func(t *testing.T) {+ t.Skip("FIXME not implemented")+ link1Name := "link1"++ run.symlink(t, ".", link1Name)+ run.mkdir(t, "dir1/link1")+ run.checkDir(t, baseState+"|link1 1|dir1/link1/")++ err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))+ assert.Error(t, err)++ run.rm(t, link1Name)+ run.rm(t, "dir1/link1")+ checkBaseState()+ })++ // Move directory -> symlink+ t.Run("MoveDirectoryToSymlink", func(t *testing.T) {+ t.Skip("FIXME not implemented")+ link1Name := "dir1/link1"++ run.mkdir(t, "link1")+ run.symlink(t, ".", link1Name)+ run.checkDir(t, baseState+"|link1/|dir1/link1 1")++ err := run.os.Rename(run.path("link1"), run.path("dir1/link1"))+ assert.Error(t, err)++ run.rm(t, "link1")+ run.rm(t, link1Name)+ checkBaseState()+ })+}diff --git a/vfs/vfstest/fs.go b/vfs/vfstest/fs.goindex 1c74f54dc..944ff4a35 100644--- a/vfs/vfstest/fs.go+++ b/vfs/vfstest/fs.go@@ -49,12 +49,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cachtests := []struct {cacheMode vfscommon.CacheModewriteBack fs.Duration+ links bool}{{cacheMode: vfscommon.CacheModeOff},+ {cacheMode: vfscommon.CacheModeOff, links: true},{cacheMode: vfscommon.CacheModeMinimal},{cacheMode: vfscommon.CacheModeWrites},{cacheMode: vfscommon.CacheModeFull},{cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond)},+ {cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond), links: true},}for _, test := range tests {if test.cacheMode < minimumRequiredCacheMode {@@ -63,11 +66,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.CachvfsOpt := vfscommon.OptvfsOpt.CacheMode = test.cacheModevfsOpt.WriteBack = test.writeBack+ vfsOpt.Links = test.linksrun = newRun(useVFS, &vfsOpt, mountFn)what := fmt.Sprintf("CacheMode=%v", test.cacheMode)if test.writeBack > 0 {what += fmt.Sprintf(",WriteBack=%v", test.writeBack)}+ if test.links {+ what += fmt.Sprintf(",Links=%v", test.links)+ }fs.Logf(nil, "Starting test run with %s", what)ok := t.Run(what, func(t *testing.T) {t.Run("TestTouchAndDelete", TestTouchAndDelete)@@ -98,6 +105,7 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cacht.Run("TestWriteFileFsync", TestWriteFileFsync)t.Run("TestWriteFileDup", TestWriteFileDup)t.Run("TestWriteFileAppend", TestWriteFileAppend)+ t.Run("TestSymlinks", TestSymlinks)})fs.Logf(nil, "Finished test run with %s (ok=%v)", what, ok)run.Finalise()@@ -213,10 +221,16 @@ func newDirMap(dirString string) (dm dirMap) {}// Returns a dirmap with only the files in-func (dm dirMap) filesOnly() dirMap {+func (dm dirMap) filesOnly(stripLinksSuffix bool) dirMap {newDm := make(dirMap)for name := range dm {if !strings.HasSuffix(name, "/") {+ if stripLinksSuffix {+ index := strings.LastIndex(name, " ")+ if index != -1 {+ name = strings.TrimSuffix(name[0:index], fs.LinkSuffix) + name[index:]+ }+ }newDm[name] = struct{}{}}}@@ -224,7 +238,9 @@ func (dm dirMap) filesOnly() dirMap {}// reads the local tree into dir-func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {+//+// If recurse it set it will recurse into subdirectories+func (r *Run) readLocalEx(t *testing.T, dir dirMap, filePath string, recurse bool) {realPath := r.path(filePath)files, err := r.os.ReadDir(realPath)require.NoError(t, err)@@ -232,15 +248,26 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {name := path.Join(filePath, fi.Name())if fi.IsDir() {dir[name+"/"] = struct{}{}- r.readLocal(t, dir, name)+ if recurse {+ r.readLocalEx(t, dir, name, recurse)+ }assert.Equal(t, os.FileMode(r.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm())} else {dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}- assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())+ if fi.Mode()&os.ModeSymlink != 0 {+ assert.Equal(t, os.FileMode(r.vfsOpt.LinkPerms)&os.ModePerm, fi.Mode().Perm())+ } else {+ assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())+ }}}}+// reads the local tree into dir+func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {+ r.readLocalEx(t, dir, filePath, true)+}+// reads the remote tree into dirfunc (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1)@@ -271,7 +298,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {remoteDm = make(dirMap)r.readRemote(t, remoteDm, "")// Ignore directories for remote compare- remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly())+ remoteOK = reflect.DeepEqual(dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links))fuseOK = reflect.DeepEqual(dm, localDm)if remoteOK && fuseOK {return@@ -280,7 +307,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)time.Sleep(sleep)}- assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")+ assert.Equal(t, dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links), "expected vs remote")assert.Equal(t, dm, localDm, "expected vs fuse mount")}@@ -353,6 +380,37 @@ func (r *Run) rmdir(t *testing.T, filepath string) {require.NoError(t, err)}+func (r *Run) symlink(t *testing.T, oldname, newname string) {+ newname = r.path(newname)+ err := r.os.Symlink(oldname, newname)+ require.NoError(t, err)+}++func (r *Run) checkMode(t *testing.T, name string, lexpected os.FileMode, expected os.FileMode) {+ if r.useVFS {+ info, err := run.os.Stat(run.path(name))+ require.NoError(t, err)+ assert.Equal(t, lexpected, info.Mode())+ assert.Equal(t, name, info.Name())+ } else {+ info, err := os.Lstat(run.path(name))+ require.NoError(t, err)+ assert.Equal(t, lexpected, info.Mode())+ assert.Equal(t, name, info.Name())++ info, err = run.os.Stat(run.path(name))+ require.NoError(t, err)+ assert.Equal(t, expected, info.Mode())+ assert.Equal(t, name, info.Name())+ }+}++func (r *Run) readlink(t *testing.T, name string) string {+ result, err := r.os.Readlink(r.path(name))+ require.NoError(t, err)+ return result+}+// TestMount checks that the Fs is mounted by seeing if the mountpoint// is in the mount outputfunc TestMount(t *testing.T) {diff --git a/vfs/vfstest/os.go b/vfs/vfstest/os.goindex 6e7a1b640..ea6d00641 100644--- a/vfs/vfstest/os.go+++ b/vfs/vfstest/os.go@@ -22,6 +22,8 @@ type Oser interface {Remove(name string) errorRename(oldName, newName string) errorStat(path string) (os.FileInfo, error)+ Symlink(oldname, newname string) error+ Readlink(name string) (s string, err error)}// realOs is an implementation of Oser backed by the "os" package@@ -130,6 +132,16 @@ func (r realOs) Stat(path string) (os.FileInfo, error) {return os.Stat(path)}+// Symlink+func (r realOs) Symlink(oldname, newname string) error {+ return os.Symlink(oldname, newname)+}++// Readlink+func (r realOs) Readlink(name string) (s string, err error) {+ return os.Readlink(name)+}+// Check interfacesvar _ Oser = &realOs{}var _ vfs.Handle = &realOsFile{}