Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

vfs: add symlink support to VFS

Nick Craig-Wooda5abe4b8b3ec
diff --git a/vfs/dir.go b/vfs/dir.go
index 26b706b62..a76d00cb1 100644
--- a/vfs/dir.go
+++ b/vfs/dir.go
@@ -459,7 +459,8 @@ func (d *Dir) addObject(node Node) {
// This will be replaced with a real object when it is read back from the
// remote.
//
-// This is used to add directory entries while things are uploading
+// This is used by the vfs cache to insert objects that are uploading
+// into the directory tree.
func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
var node Node
d.mu.RLock()
@@ -475,7 +476,16 @@ func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
entry := fs.NewDir(remote, time.Now())
node = newDir(d.vfs, d.f, d, entry)
} else {
+ isLink := false
+ if d.vfs.Opt.Links {
+ // since the path came from the cache it may have fs.LinkSuffix,
+ // so remove it and mark the *File accordingly
+ leaf, isLink = strings.CutSuffix(leaf, fs.LinkSuffix)
+ }
f := newFile(d, dPath, nil, leaf)
+ if isLink {
+ f.setSymlink()
+ }
f.setSize(size)
node = f
}
@@ -628,7 +638,7 @@ func (d *Dir) _purgeVirtual() {
// if writing in progress then leave virtual
continue
}
- if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.Path()) {
+ if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.CachePath()) {
// if object in use or dirty then leave virtual
continue
}
@@ -718,6 +728,9 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
if name == "." || name == ".." {
continue
}
+ if d.vfs.Opt.Links {
+ name, _ = strings.CutSuffix(name, fs.LinkSuffix)
+ }
node := d.items[name]
if mv.add(d, name) {
continue
diff --git a/vfs/file.go b/vfs/file.go
index 1c599ef01..b94e3dbb4 100644
--- a/vfs/file.go
+++ b/vfs/file.go
@@ -4,8 +4,10 @@ import (
"context"
"errors"
"fmt"
+ "io"
"os"
"path"
+ "strings"
"sync"
"sync/atomic"
"time"
@@ -34,7 +36,7 @@ import (
//
// File may **not** read any members of Dir directly.
-// File represents a file
+// File represents a file or a symlink
type File struct {
inode uint64 // inode number - read only
size atomic.Int64 // size of file
@@ -53,6 +55,7 @@ type File struct {
sys atomic.Value // user defined info to be attached here
nwriters atomic.Int32 // len(writers)
appendMode bool // file was opened with O_APPEND
+ isLink bool // file represents a symlink
}
// newFile creates a new File
@@ -69,9 +72,18 @@ func newFile(d *Dir, dPath string, o fs.Object, leaf string) *File {
if o != nil {
f.size.Store(o.Size())
}
+ f._setIsLink()
return f
}
+// Set whether this is a link or not based on f.o
+func (f *File) _setIsLink() {
+ if f.o == nil {
+ return
+ }
+ f.isLink = f.d.vfs.Opt.Links && strings.HasSuffix(f.o.Remote(), fs.LinkSuffix)
+}
+
// String converts it to printable
func (f *File) String() string {
if f == nil {
@@ -90,13 +102,31 @@ func (f *File) IsDir() bool {
return false
}
+// IsSymlink returns true for symlinks when --links is enabled
+func (f *File) IsSymlink() bool {
+ f.mu.RLock()
+ defer f.mu.RUnlock()
+ return f.isLink
+}
+
+// setSymlink marks this File as being a symlink
+func (f *File) setSymlink() {
+ f.mu.RLock()
+ f.isLink = true
+ f.mu.RUnlock()
+}
+
// Mode bits of the file or directory - satisfies Node interface
func (f *File) Mode() (mode os.FileMode) {
f.mu.RLock()
defer f.mu.RUnlock()
- mode = os.FileMode(f.d.vfs.Opt.FilePerms)
- if f.appendMode {
- mode |= os.ModeAppend
+ if f.isLink {
+ mode = os.FileMode(f.d.vfs.Opt.LinkPerms)
+ } else {
+ mode = os.FileMode(f.d.vfs.Opt.FilePerms)
+ if f.appendMode {
+ mode |= os.ModeAppend
+ }
}
return mode
}
@@ -122,6 +152,34 @@ func (f *File) Path() string {
return path.Join(dPath, leaf)
}
+// _fixCachePath returns fullPath with the fs.LinkSuffix added if appropriate
+// use when lock is held
+func (f *File) _fixCachePath(fullPath string) string {
+ if !f.isLink {
+ return fullPath
+ }
+ return fullPath + fs.LinkSuffix
+}
+
+// _cachePath returns the full path of the file with the fs.LinkSuffix if appropriate
+// use when lock is held
+func (f *File) _cachePath() string {
+ dPath, leaf := f.dPath, f.leaf
+ if f.isLink {
+ leaf += fs.LinkSuffix
+ }
+ return path.Join(dPath, leaf)
+}
+
+// CachePath returns the full path of the file with the fs.LinkSuffix if appropriate
+//
+// We use this path when storing files in the cache.
+func (f *File) CachePath() string {
+ f.mu.RLock()
+ defer f.mu.RUnlock()
+ return f._cachePath()
+}
+
// Sys returns underlying data source (can be nil) - satisfies Node interface
func (f *File) Sys() interface{} {
return f.sys.Load()
@@ -172,6 +230,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
f.mu.RLock()
d := f.d
oldPendingRenameFun := f.pendingRenameFun
+ oldPath := f._cachePath()
+ newCacheName := f._fixCachePath(newName)
f.mu.RUnlock()
if features := d.Fs().Features(); features.Move == nil && features.Copy == nil {
@@ -180,9 +240,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
return err
}
- oldPath := f.Path()
// File.mu is unlocked here to call Dir.Path()
- newPath := path.Join(destDir.Path(), newName)
+ newPath := path.Join(destDir.Path(), newCacheName)
renameCall := func(ctx context.Context) (err error) {
// chain rename calls if any
@@ -231,6 +290,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
f.mu.Lock()
if newObject != nil {
f.o = newObject
+ f._setIsLink()
}
f.pendingRenameFun = nil
f.mu.Unlock()
@@ -334,7 +394,7 @@ func (f *File) ModTime() (modTime time.Time) {
}
// Read the modtime from a dirty item if it exists
if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
- if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
+ if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {
modTime, err := item.GetModTime()
if err != nil {
fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)
@@ -371,7 +431,7 @@ func (f *File) Size() int64 {
// Read the size from a dirty item if it exists
if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
- if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
+ if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {
size, err := item.GetSize()
if err != nil {
fs.Errorf(f._path(), "Size: Item GetSize failed: %v", err)
@@ -404,8 +464,8 @@ func (f *File) SetModTime(modTime time.Time) error {
f.pendingModTime = modTime
// set the time of the file in the cache
- if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) {
- f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
+ if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._cachePath()) {
+ f.d.vfs.cache.SetModTime(f._cachePath(), f.pendingModTime)
}
// Only update the ModTime when there are no writers, setObject will do it
@@ -480,6 +540,7 @@ func (f *File) setSize(n int64) {
func (f *File) setObject(o fs.Object) {
f.mu.Lock()
f.o = o
+ f._setIsLink()
_ = f._applyPendingModTime()
d := f.d
f.mu.Unlock()
@@ -493,6 +554,7 @@ func (f *File) setObject(o fs.Object) {
func (f *File) setObjectNoUpdate(o fs.Object) {
f.mu.Lock()
f.o = o
+ f._setIsLink()
f.virtualModTime = nil
fs.Debugf(f._path(), "Reset virtual modtime")
f.mu.Unlock()
@@ -611,8 +673,8 @@ func (f *File) Remove() (err error) {
// Remove the object from the cache
wasWriting := false
- if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) {
- wasWriting = d.vfs.cache.Remove(f.Path())
+ if d.vfs.cache != nil && d.vfs.cache.Exists(f.CachePath()) {
+ wasWriting = d.vfs.cache.Remove(f.CachePath())
}
f.muRW.Lock() // muRW must be locked before mu to avoid
@@ -673,6 +735,85 @@ func (f *File) Fs() fs.Fs {
return f.d.Fs()
}
+// MaxSymlinkIterations is the largest number of symlink evaluations EvalSymlinks will do.
+const MaxSymlinkIterations = 32
+
+// If f is a symlink then it resolves it to a new Node.
+//
+// This is a simplistic symlink resolver - it only resolves direct
+// symlinks, it will **not** resolve paths that point into a directory
+// via a symlink.
+//
+// It returns the target node after the evaluation of all symbolic
+// links.
+//
+// It returns an error if too many symlinks need to be resolved
+// (ELOOP) or there is a loop.
+func (f *File) resolveNode() (target Node, err error) {
+ defer log.Trace(f.Path(), "")("target=%v, err=%v", &target, &err)
+ seen := make(map[string]struct{})
+ for tries := 0; tries < MaxSymlinkIterations; tries++ {
+ // If f isn't a symlink, we've arrived at the target
+ if !f.IsSymlink() {
+ return f, nil
+ }
+
+ // Read the symlink
+ fd, err := f.Open(os.O_RDONLY | o_SYMLINK)
+ if err != nil {
+ return nil, err
+ }
+ b, err := io.ReadAll(fd)
+ closeErr := fd.Close()
+ if err != nil {
+ return nil, err
+ }
+ if closeErr != nil {
+ return nil, closeErr
+ }
+ targetPath := string(b)
+
+ // Convert to a path relative to the root
+ // Symlinks are relative to their file node
+ if !path.IsAbs(targetPath) {
+ basePath := path.Dir(f.Path())
+ targetPath = path.Join(basePath, targetPath)
+ }
+
+ // Clean the path, rclone style
+ targetPath = path.Clean(targetPath)
+ if targetPath == "." {
+ targetPath = ""
+ }
+
+ // Check if we've already seen this path
+ if _, ok := seen[targetPath]; ok {
+ return nil, ELOOP
+ }
+ seen[targetPath] = struct{}{}
+
+ // Resolve the targetPath into a node
+ target, err = f.d.vfs.Stat(targetPath)
+ if err != nil {
+ return nil, err
+ }
+
+ // Return node as it must be the destination if not a file
+ var ok bool
+ f, ok = target.(*File)
+ if !ok {
+ return target, nil
+ }
+ }
+ return nil, ELOOP
+}
+
+// Open also also implements the internal flag o_SYMLINK which instead
+// of opening the file a symlink points to, opens the symlink itself.
+// This is used for reading and writing the symlink and shouldn't be
+// used externally.
+const o_SYMLINK = 0x4000_0000 //nolint:revive
+
// Open a file according to the flags provided
//
// O_RDONLY open the file read-only.
@@ -694,6 +835,16 @@ func (f *File) Open(flags int) (fd Handle, err error) {
rdwrMode = flags & accessModeMask
)
+ // If this is a symlink, then resolve it
+ if f.IsSymlink() && flags&o_SYMLINK == 0 {
+ target, err := f.resolveNode()
+ if err != nil {
+ return nil, err
+ }
+ return target.Open(flags)
+ }
+ flags &^= o_SYMLINK
+
// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html
// The result of using O_TRUNC with O_RDONLY is undefined.
// Linux seems to truncate the file, but we prefer to return EINVAL
@@ -738,7 +889,7 @@ func (f *File) Open(flags int) (fd Handle, err error) {
d := f.d
f.mu.RUnlock()
CacheMode := d.vfs.Opt.CacheMode
- if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.Path()) || d.vfs.cache.Exists(f.Path())) {
+ if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.CachePath()) || d.vfs.cache.Exists(f.CachePath())) {
fd, err = f.openRW(flags)
} else if read && write {
if CacheMode >= vfscommon.CacheModeMinimal {
diff --git a/vfs/read_write.go b/vfs/read_write.go
index 4130e68b0..56b9c467d 100644
--- a/vfs/read_write.go
+++ b/vfs/read_write.go
@@ -43,7 +43,7 @@ func (fh *RWFileHandle) Unlock() error {
func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
defer log.Trace(f.Path(), "")("err=%v", &err)
// get an item to represent this from the cache
- item := d.vfs.cache.Item(f.Path())
+ item := d.vfs.cache.Item(f.CachePath())
exists := f.exists() || (item.Exists() && !item.WrittenBack())
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 2e60c8fb6..aa6fe15eb 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -791,6 +791,9 @@ func (vfs *VFS) WriteFile(name string, data []byte, perm os.FileMode) (err error
}
// AddVirtual adds the object (file or dir) to the directory cache
+//
+// This is used by the vfs cache to insert objects that are uploading
+// into the directory tree.
func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {
remote = strings.TrimRight(remote, "/")
var dir *Dir
@@ -808,3 +811,85 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {
dir.AddVirtual(leaf, size, false)
return nil
}
+
+// Readlink returns the destination of the named symbolic link.
+// If there is an error, it will be of type *PathError.
+func (vfs *VFS) Readlink(name string) (s string, err error) {
+ if !vfs.Opt.Links {
+ fs.Errorf(nil, "symlinks not supported without the --links flag: %v", name)
+ return "", ENOSYS
+ }
+ node, err := vfs.Stat(name)
+ if err != nil {
+ return "", err
+ }
+ file, ok := node.(*File)
+ if !ok || !file.IsSymlink() {
+ return "", EINVAL // not a symlink
+ }
+ fd, err := file.Open(os.O_RDONLY | o_SYMLINK)
+ if err != nil {
+ return "", err
+ }
+ defer fs.CheckClose(fd, &err)
+ b, err := io.ReadAll(fd)
+ if err != nil {
+ return "", err
+ }
+ return string(b), nil
+}
+
+// CreateSymlink creates newname as a symbolic link to oldname.
+// On Windows, a symlink to a non-existent oldname creates a file symlink;
+// if oldname is later created as a directory the symlink will not work.
+// It returns the node created
+func (vfs *VFS) CreateSymlink(oldname, newname string) (Node, error) {
+ if !vfs.Opt.Links {
+ fs.Errorf(newname, "symlinks not supported without the --links flag")
+ return nil, ENOSYS
+ }
+
+ // Destination can't exist
+ _, err := vfs.Stat(newname)
+ if err == nil {
+ return nil, EEXIST
+ } else if err != ENOENT {
+ return nil, err
+ }
+
+ // Find the parent
+ dir, leaf, err := vfs.StatParent(newname)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create the file node
+ flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | o_SYMLINK
+ file, err := dir.Create(leaf, flags)
+ if err != nil {
+ return nil, err
+ }
+
+ // Force the file to be a link
+ file.setSymlink()
+
+ // Open the file
+ fh, err := file.Open(flags)
+ if err != nil {
+ return nil, err
+ }
+ defer fs.CheckClose(fh, &err)
+
+ // Write the symlink data
+ _, err = fh.Write([]byte(strings.ReplaceAll(oldname, "\\", "/")))
+
+ return file, nil
+}
+
+// Symlink creates newname as a symbolic link to oldname.
+// On Windows, a symlink to a non-existent oldname creates a file symlink;
+// if oldname is later created as a directory the symlink will not work.
+func (vfs *VFS) Symlink(oldname, newname string) error {
+ _, err := vfs.CreateSymlink(oldname, newname)
+ return err
+}
diff --git a/vfs/write.go b/vfs/write.go
index c799f98d5..8e74b45a8 100644
--- a/vfs/write.go
+++ b/vfs/write.go
@@ -37,6 +37,9 @@ var (
)
func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHandle, error) {
+ if f.IsSymlink() {
+ remote += fs.LinkSuffix
+ }
fh := &WriteFileHandle{
remote: remote,
flags: flags,