Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

vfstest: make VFS test suite support symlinks

Filipe Azevedof1d2f2b2c855
diff --git a/vfs/vfstest/file.go b/vfs/vfstest/file.go
index a4e490f0e..e923b521f 100644
--- a/vfs/vfstest/file.go
+++ b/vfs/vfstest/file.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -72,3 +73,230 @@ func TestFileModTimeWithOpenWriters(t *testing.T) {
run.rm(t, "cp-archive-test")
}
+
+// TestSymlinks tests all the api of the VFS / Mount symlinks support
+func TestSymlinks(t *testing.T) {
+ run.skipIfNoFUSE(t)
+ if !run.vfsOpt.Links {
+ t.Skip("No symlinks configured")
+ }
+
+ if runtime.GOOS == "windows" {
+ t.Skip("Skipping test on Windows")
+ }
+
+ fs.Logf(nil, "Links: %v, useVFS: %v, suffix: %v", run.vfsOpt.Links, run.useVFS, fs.LinkSuffix)
+
+ // Create initial setup of test files and directories we will create links to
+ run.mkdir(t, "dir1")
+ run.mkdir(t, "dir1/sub1dir1")
+ run.createFile(t, "dir1/file1", "potato")
+ run.mkdir(t, "dir2")
+ run.mkdir(t, "dir2/sub1dir2")
+ run.createFile(t, "dir2/file1", "chicken")
+
+ // base state all the tests will be build off
+ baseState := "dir1/|dir1/sub1dir1/|dir1/file1 6|dir2/|dir2/sub1dir2/|dir2/file1 7"
+ // Check the tests return to the base state
+ checkBaseState := func() {
+ run.checkDir(t, baseState)
+ }
+ checkBaseState()
+
+ t.Run("FileLink", func(t *testing.T) {
+ // Link to a file
+ run.symlink(t, "dir1/file1", "dir1file1_link")
+ run.checkDir(t, baseState+"|dir1file1_link 10")
+ run.checkMode(t, "dir1file1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.FilePerms))
+ assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link"))
+
+ // Read through a symlink
+ assert.Equal(t, "potato", run.readFile(t, "dir1file1_link"))
+
+ // Write through a symlink
+ err := writeFile(run.path("dir1file1_link"), []byte("carrot"), 0600)
+ require.NoError(t, err)
+ assert.Equal(t, "carrot", run.readFile(t, "dir1file1_link"))
+ assert.Equal(t, "carrot", run.readFile(t, "dir1/file1"))
+
+ // Rename a symlink
+ err = run.os.Rename(run.path("dir1file1_link"), run.path("dir1file1_link")+"_bla")
+ require.NoError(t, err)
+ run.checkDir(t, baseState+"|dir1file1_link_bla 10")
+ assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link_bla"))
+
+ // Delete a symlink
+ run.rm(t, "dir1file1_link_bla")
+ checkBaseState()
+ })
+
+ t.Run("DirLink", func(t *testing.T) {
+ // Link to a dir
+ run.symlink(t, "dir1", "dir1_link")
+ run.checkDir(t, baseState+"|dir1_link 4")
+ run.checkMode(t, "dir1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.DirPerms))
+ assert.Equal(t, "dir1", run.readlink(t, "dir1_link"))
+
+ // Check you can't open a directory symlink
+ _, err := run.os.OpenFile(run.path("dir1_link"), os.O_WRONLY, 0600)
+ require.Error(t, err)
+
+ // Our symlink resolution is very simple when using the VFS as when using the
+ // mount the OS will resolve the symlinks, so we don't recurse here
+
+ // Read entries directly
+ dir1Entries := make(dirMap)
+ run.readLocalEx(t, dir1Entries, "dir1", false)
+ assert.Equal(t, newDirMap("dir1/sub1dir1/|dir1/file1 6"), dir1Entries)
+
+ // Read entries through the directory symlink
+ dir1EntriesSymlink := make(dirMap)
+ run.readLocalEx(t, dir1EntriesSymlink, "dir1_link", false)
+ assert.Equal(t, newDirMap("dir1_link/sub1dir1/|dir1_link/file1 6"), dir1EntriesSymlink)
+
+ // Rename directory symlink
+ err = run.os.Rename(run.path("dir1_link"), run.path("dir1_link")+"_bla")
+ require.NoError(t, err)
+ run.checkDir(t, baseState+"|dir1_link_bla 4")
+ assert.Equal(t, "dir1", run.readlink(t, "dir1_link_bla"))
+
+ // Remove directory symlink
+ run.rm(t, "dir1_link_bla")
+
+ checkBaseState()
+ })
+
+ // Corner case #1 - We do not allow creating regular and symlink files having the same name (ie, test.txt and test.txt.rclonelink)
+
+ // Symlink first, then regular
+ t.Run("OverwriteSymlinkWithRegular", func(t *testing.T) {
+ link1Name := "link1.txt"
+
+ run.symlink(t, "dir1/file1", link1Name)
+ run.checkDir(t, baseState+"|link1.txt 10")
+
+ fh, err := run.os.OpenFile(run.path(link1Name), os.O_WRONLY|os.O_CREATE, os.FileMode(run.vfsOpt.FilePerms))
+
+ // On real mount with links enabled, that open the symlink target as expected, else that fails to create a new file
+ assert.NoError(t, err)
+ // Don't care about the result, in some cache mode the file can't be opened for writing, so closing would trigger an err
+ _ = fh.Close()
+
+ run.rm(t, link1Name)
+ checkBaseState()
+ })
+
+ // Regular first, then symlink
+ t.Run("OverwriteRegularWithSymlink", func(t *testing.T) {
+ link1Name := "link1.txt"
+
+ run.createFile(t, link1Name, "")
+ run.checkDir(t, baseState+"|link1.txt 0")
+
+ err := run.os.Symlink(".", run.path(link1Name))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ checkBaseState()
+ })
+
+ // Corner case #2 - We do not allow creating directory and symlink file having the same name (ie, test and test.rclonelink)
+
+ // Symlink first, then directory
+ t.Run("OverwriteSymlinkWithDirectory", func(t *testing.T) {
+ link1Name := "link1"
+
+ run.symlink(t, ".", link1Name)
+ run.checkDir(t, baseState+"|link1 1")
+
+ err := run.os.Mkdir(run.path(link1Name), os.FileMode(run.vfsOpt.DirPerms))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ checkBaseState()
+ })
+
+ // Directory first, then symlink
+ t.Run("OverwriteDirectoryWithSymlink", func(t *testing.T) {
+ link1Name := "link1"
+
+ run.mkdir(t, link1Name)
+ run.checkDir(t, baseState+"|link1/")
+
+ err := run.os.Symlink(".", run.path(link1Name))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ checkBaseState()
+ })
+
+ // Corner case #3 - We do not allow moving directory or file having the same name in a target (ie, test and test.rclonelink)
+
+ // Move symlink -> regular file
+ t.Run("MoveSymlinkToFile", func(t *testing.T) {
+ t.Skip("FIXME not implemented")
+ link1Name := "link1.txt"
+
+ run.symlink(t, ".", link1Name)
+ run.createFile(t, "dir1/link1.txt", "")
+ run.checkDir(t, baseState+"|link1.txt 1|dir1/link1.txt 0")
+
+ err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ run.rm(t, "dir1/link1.txt")
+ checkBaseState()
+ })
+
+ // Move regular file -> symlink
+ t.Run("MoveFileToSymlink", func(t *testing.T) {
+ t.Skip("FIXME not implemented")
+ link1Name := "link1.txt"
+
+ run.createFile(t, link1Name, "")
+ run.symlink(t, ".", "dir1/"+link1Name)
+ run.checkDir(t, baseState+"|link1.txt 0|dir1/link1.txt 1")
+
+ err := run.os.Rename(run.path(link1Name), run.path("dir1/link1.txt"))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ run.rm(t, "dir1/"+link1Name)
+ checkBaseState()
+ })
+
+ // Move symlink -> directory
+ t.Run("MoveSymlinkToDirectory", func(t *testing.T) {
+ t.Skip("FIXME not implemented")
+ link1Name := "link1"
+
+ run.symlink(t, ".", link1Name)
+ run.mkdir(t, "dir1/link1")
+ run.checkDir(t, baseState+"|link1 1|dir1/link1/")
+
+ err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name))
+ assert.Error(t, err)
+
+ run.rm(t, link1Name)
+ run.rm(t, "dir1/link1")
+ checkBaseState()
+ })
+
+ // Move directory -> symlink
+ t.Run("MoveDirectoryToSymlink", func(t *testing.T) {
+ t.Skip("FIXME not implemented")
+ link1Name := "dir1/link1"
+
+ run.mkdir(t, "link1")
+ run.symlink(t, ".", link1Name)
+ run.checkDir(t, baseState+"|link1/|dir1/link1 1")
+
+ err := run.os.Rename(run.path("link1"), run.path("dir1/link1"))
+ assert.Error(t, err)
+
+ run.rm(t, "link1")
+ run.rm(t, link1Name)
+ checkBaseState()
+ })
+}
diff --git a/vfs/vfstest/fs.go b/vfs/vfstest/fs.go
index 1c74f54dc..944ff4a35 100644
--- a/vfs/vfstest/fs.go
+++ b/vfs/vfstest/fs.go
@@ -49,12 +49,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
tests := []struct {
cacheMode vfscommon.CacheMode
writeBack fs.Duration
+ links bool
}{
{cacheMode: vfscommon.CacheModeOff},
+ {cacheMode: vfscommon.CacheModeOff, links: true},
{cacheMode: vfscommon.CacheModeMinimal},
{cacheMode: vfscommon.CacheModeWrites},
{cacheMode: vfscommon.CacheModeFull},
{cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond)},
+ {cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond), links: true},
}
for _, test := range tests {
if test.cacheMode < minimumRequiredCacheMode {
@@ -63,11 +66,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
vfsOpt := vfscommon.Opt
vfsOpt.CacheMode = test.cacheMode
vfsOpt.WriteBack = test.writeBack
+ vfsOpt.Links = test.links
run = newRun(useVFS, &vfsOpt, mountFn)
what := fmt.Sprintf("CacheMode=%v", test.cacheMode)
if test.writeBack > 0 {
what += fmt.Sprintf(",WriteBack=%v", test.writeBack)
}
+ if test.links {
+ what += fmt.Sprintf(",Links=%v", test.links)
+ }
fs.Logf(nil, "Starting test run with %s", what)
ok := t.Run(what, func(t *testing.T) {
t.Run("TestTouchAndDelete", TestTouchAndDelete)
@@ -98,6 +105,7 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
t.Run("TestWriteFileFsync", TestWriteFileFsync)
t.Run("TestWriteFileDup", TestWriteFileDup)
t.Run("TestWriteFileAppend", TestWriteFileAppend)
+ t.Run("TestSymlinks", TestSymlinks)
})
fs.Logf(nil, "Finished test run with %s (ok=%v)", what, ok)
run.Finalise()
@@ -213,10 +221,16 @@ func newDirMap(dirString string) (dm dirMap) {
}
// Returns a dirmap with only the files in
-func (dm dirMap) filesOnly() dirMap {
+func (dm dirMap) filesOnly(stripLinksSuffix bool) dirMap {
newDm := make(dirMap)
for name := range dm {
if !strings.HasSuffix(name, "/") {
+ if stripLinksSuffix {
+ index := strings.LastIndex(name, " ")
+ if index != -1 {
+ name = strings.TrimSuffix(name[0:index], fs.LinkSuffix) + name[index:]
+ }
+ }
newDm[name] = struct{}{}
}
}
@@ -224,7 +238,9 @@ func (dm dirMap) filesOnly() dirMap {
}
// reads the local tree into dir
-func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
+//
+// If recurse it set it will recurse into subdirectories
+func (r *Run) readLocalEx(t *testing.T, dir dirMap, filePath string, recurse bool) {
realPath := r.path(filePath)
files, err := r.os.ReadDir(realPath)
require.NoError(t, err)
@@ -232,15 +248,26 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
name := path.Join(filePath, fi.Name())
if fi.IsDir() {
dir[name+"/"] = struct{}{}
- r.readLocal(t, dir, name)
+ if recurse {
+ r.readLocalEx(t, dir, name, recurse)
+ }
assert.Equal(t, os.FileMode(r.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm())
} else {
dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
- assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())
+ if fi.Mode()&os.ModeSymlink != 0 {
+ assert.Equal(t, os.FileMode(r.vfsOpt.LinkPerms)&os.ModePerm, fi.Mode().Perm())
+ } else {
+ assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())
+ }
}
}
}
+// reads the local tree into dir
+func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
+ r.readLocalEx(t, dir, filePath, true)
+}
+
// reads the remote tree into dir
func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1)
@@ -271,7 +298,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {
remoteDm = make(dirMap)
r.readRemote(t, remoteDm, "")
// Ignore directories for remote compare
- remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly())
+ remoteOK = reflect.DeepEqual(dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links))
fuseOK = reflect.DeepEqual(dm, localDm)
if remoteOK && fuseOK {
return
@@ -280,7 +307,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) {
t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
time.Sleep(sleep)
}
- assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")
+ assert.Equal(t, dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links), "expected vs remote")
assert.Equal(t, dm, localDm, "expected vs fuse mount")
}
@@ -353,6 +380,37 @@ func (r *Run) rmdir(t *testing.T, filepath string) {
require.NoError(t, err)
}
+func (r *Run) symlink(t *testing.T, oldname, newname string) {
+ newname = r.path(newname)
+ err := r.os.Symlink(oldname, newname)
+ require.NoError(t, err)
+}
+
+func (r *Run) checkMode(t *testing.T, name string, lexpected os.FileMode, expected os.FileMode) {
+ if r.useVFS {
+ info, err := run.os.Stat(run.path(name))
+ require.NoError(t, err)
+ assert.Equal(t, lexpected, info.Mode())
+ assert.Equal(t, name, info.Name())
+ } else {
+ info, err := os.Lstat(run.path(name))
+ require.NoError(t, err)
+ assert.Equal(t, lexpected, info.Mode())
+ assert.Equal(t, name, info.Name())
+
+ info, err = run.os.Stat(run.path(name))
+ require.NoError(t, err)
+ assert.Equal(t, expected, info.Mode())
+ assert.Equal(t, name, info.Name())
+ }
+}
+
+func (r *Run) readlink(t *testing.T, name string) string {
+ result, err := r.os.Readlink(r.path(name))
+ require.NoError(t, err)
+ return result
+}
+
// TestMount checks that the Fs is mounted by seeing if the mountpoint
// is in the mount output
func TestMount(t *testing.T) {
diff --git a/vfs/vfstest/os.go b/vfs/vfstest/os.go
index 6e7a1b640..ea6d00641 100644
--- a/vfs/vfstest/os.go
+++ b/vfs/vfstest/os.go
@@ -22,6 +22,8 @@ type Oser interface {
Remove(name string) error
Rename(oldName, newName string) error
Stat(path string) (os.FileInfo, error)
+ Symlink(oldname, newname string) error
+ Readlink(name string) (s string, err error)
}
// realOs is an implementation of Oser backed by the "os" package
@@ -130,6 +132,16 @@ func (r realOs) Stat(path string) (os.FileInfo, error) {
return os.Stat(path)
}
+// Symlink
+func (r realOs) Symlink(oldname, newname string) error {
+ return os.Symlink(oldname, newname)
+}
+
+// Readlink
+func (r realOs) Readlink(name string) (s string, err error) {
+ return os.Readlink(name)
+}
+
// Check interfaces
var _ Oser = &realOs{}
var _ vfs.Handle = &realOsFile{}