"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: add symlink support to VFS
Nick Craig-Wooda5abe4b8b3ec
diff --git a/vfs/dir.go b/vfs/dir.goindex 26b706b62..a76d00cb1 100644--- a/vfs/dir.go+++ b/vfs/dir.go@@ -459,7 +459,8 @@ func (d *Dir) addObject(node Node) {// This will be replaced with a real object when it is read back from the// remote.//-// This is used to add directory entries while things are uploading+// This is used by the vfs cache to insert objects that are uploading+// into the directory tree.func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {var node Noded.mu.RLock()@@ -475,7 +476,16 @@ func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {entry := fs.NewDir(remote, time.Now())node = newDir(d.vfs, d.f, d, entry)} else {+ isLink := false+ if d.vfs.Opt.Links {+ // since the path came from the cache it may have fs.LinkSuffix,+ // so remove it and mark the *File accordingly+ leaf, isLink = strings.CutSuffix(leaf, fs.LinkSuffix)+ }f := newFile(d, dPath, nil, leaf)+ if isLink {+ f.setSymlink()+ }f.setSize(size)node = f}@@ -628,7 +638,7 @@ func (d *Dir) _purgeVirtual() {// if writing in progress then leave virtualcontinue}- if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.Path()) {+ if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.CachePath()) {// if object in use or dirty then leave virtualcontinue}@@ -718,6 +728,9 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTreeif name == "." || name == ".." {continue}+ if d.vfs.Opt.Links {+ name, _ = strings.CutSuffix(name, fs.LinkSuffix)+ }node := d.items[name]if mv.add(d, name) {continuediff --git a/vfs/file.go b/vfs/file.goindex 1c599ef01..b94e3dbb4 100644--- a/vfs/file.go+++ b/vfs/file.go@@ -4,8 +4,10 @@ import ("context""errors""fmt"+ "io""os""path"+ "strings""sync""sync/atomic""time"@@ -34,7 +36,7 @@ import (//// File may **not** read any members of Dir directly.-// File represents a file+// File represents a file or a symlinktype File struct {inode uint64 // inode number - read onlysize atomic.Int64 // size of file@@ -53,6 +55,7 @@ type File struct {sys atomic.Value // user defined info to be attached herenwriters atomic.Int32 // len(writers)appendMode bool // file was opened with O_APPEND+ isLink bool // file represents a symlink}// newFile creates a new File@@ -69,9 +72,18 @@ func newFile(d *Dir, dPath string, o fs.Object, leaf string) *File {if o != nil {f.size.Store(o.Size())}+ f._setIsLink()return f}+// Set whether this is a link or not based on f.o+func (f *File) _setIsLink() {+ if f.o == nil {+ return+ }+ f.isLink = f.d.vfs.Opt.Links && strings.HasSuffix(f.o.Remote(), fs.LinkSuffix)+}+// String converts it to printablefunc (f *File) String() string {if f == nil {@@ -90,13 +102,31 @@ func (f *File) IsDir() bool {return false}+// IsSymlink returns true for symlinks when --links is enabled+func (f *File) IsSymlink() bool {+ f.mu.RLock()+ defer f.mu.RUnlock()+ return f.isLink+}++// setSymlink marks this File as being a symlink+func (f *File) setSymlink() {+ f.mu.RLock()+ f.isLink = true+ f.mu.RUnlock()+}+// Mode bits of the file or directory - satisfies Node interfacefunc (f *File) Mode() (mode os.FileMode) {f.mu.RLock()defer f.mu.RUnlock()- mode = os.FileMode(f.d.vfs.Opt.FilePerms)- if f.appendMode {- mode |= os.ModeAppend+ if f.isLink {+ mode = os.FileMode(f.d.vfs.Opt.LinkPerms)+ } else {+ mode = os.FileMode(f.d.vfs.Opt.FilePerms)+ if f.appendMode {+ mode |= os.ModeAppend+ }}return mode}@@ -122,6 +152,34 @@ func (f *File) Path() string {return path.Join(dPath, leaf)}+// _fixCachePath returns fullPath with the fs.LinkSuffix added if appropriate+// use when lock is held+func (f *File) _fixCachePath(fullPath string) string {+ if !f.isLink {+ return fullPath+ }+ return fullPath + fs.LinkSuffix+}++// _cachePath returns the full path of the file with the fs.LinkSuffix if appropriate+// use when lock is held+func (f *File) _cachePath() string {+ dPath, leaf := f.dPath, f.leaf+ if f.isLink {+ leaf += fs.LinkSuffix+ }+ return path.Join(dPath, leaf)+}++// CachePath returns the full path of the file with the fs.LinkSuffix if appropriate+//+// We use this path when storing files in the cache.+func (f *File) CachePath() string {+ f.mu.RLock()+ defer f.mu.RUnlock()+ return f._cachePath()+}+// Sys returns underlying data source (can be nil) - satisfies Node interfacefunc (f *File) Sys() interface{} {return f.sys.Load()@@ -172,6 +230,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {f.mu.RLock()d := f.doldPendingRenameFun := f.pendingRenameFun+ oldPath := f._cachePath()+ newCacheName := f._fixCachePath(newName)f.mu.RUnlock()if features := d.Fs().Features(); features.Move == nil && features.Copy == nil {@@ -180,9 +240,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {return err}- oldPath := f.Path()// File.mu is unlocked here to call Dir.Path()- newPath := path.Join(destDir.Path(), newName)+ newPath := path.Join(destDir.Path(), newCacheName)renameCall := func(ctx context.Context) (err error) {// chain rename calls if any@@ -231,6 +290,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {f.mu.Lock()if newObject != nil {f.o = newObject+ f._setIsLink()}f.pendingRenameFun = nilf.mu.Unlock()@@ -334,7 +394,7 @@ func (f *File) ModTime() (modTime time.Time) {}// Read the modtime from a dirty item if it existsif f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {- if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {+ if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {modTime, err := item.GetModTime()if err != nil {fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)@@ -371,7 +431,7 @@ func (f *File) Size() int64 {// Read the size from a dirty item if it existsif f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {- if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {+ if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {size, err := item.GetSize()if err != nil {fs.Errorf(f._path(), "Size: Item GetSize failed: %v", err)@@ -404,8 +464,8 @@ func (f *File) SetModTime(modTime time.Time) error {f.pendingModTime = modTime// set the time of the file in the cache- if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) {- f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)+ if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._cachePath()) {+ f.d.vfs.cache.SetModTime(f._cachePath(), f.pendingModTime)}// Only update the ModTime when there are no writers, setObject will do it@@ -480,6 +540,7 @@ func (f *File) setSize(n int64) {func (f *File) setObject(o fs.Object) {f.mu.Lock()f.o = o+ f._setIsLink()_ = f._applyPendingModTime()d := f.df.mu.Unlock()@@ -493,6 +554,7 @@ func (f *File) setObject(o fs.Object) {func (f *File) setObjectNoUpdate(o fs.Object) {f.mu.Lock()f.o = o+ f._setIsLink()f.virtualModTime = nilfs.Debugf(f._path(), "Reset virtual modtime")f.mu.Unlock()@@ -611,8 +673,8 @@ func (f *File) Remove() (err error) {// Remove the object from the cachewasWriting := false- if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) {- wasWriting = d.vfs.cache.Remove(f.Path())+ if d.vfs.cache != nil && d.vfs.cache.Exists(f.CachePath()) {+ wasWriting = d.vfs.cache.Remove(f.CachePath())}f.muRW.Lock() // muRW must be locked before mu to avoid@@ -673,6 +735,85 @@ func (f *File) Fs() fs.Fs {return f.d.Fs()}+// MaxSymlinkIterations is the largest number of symlink evaluations EvalSymlinks will do.+const MaxSymlinkIterations = 32++// If f is a symlink then it resolves it to a new Node.+//+// This is a simplistic symlink resolver - it only resolves direct+// symlinks, it will **not** resolve paths that point into a directory+// via a symlink.+//+// It returns the target node after the evaluation of all symbolic+// links.+//+// It returns an error if too many symlinks need to be resolved+// (ELOOP) or there is a loop.+func (f *File) resolveNode() (target Node, err error) {+ defer log.Trace(f.Path(), "")("target=%v, err=%v", &target, &err)+ seen := make(map[string]struct{})+ for tries := 0; tries < MaxSymlinkIterations; tries++ {+ // If f isn't a symlink, we've arrived at the target+ if !f.IsSymlink() {+ return f, nil+ }++ // Read the symlink+ fd, err := f.Open(os.O_RDONLY | o_SYMLINK)+ if err != nil {+ return nil, err+ }+ b, err := io.ReadAll(fd)+ closeErr := fd.Close()+ if err != nil {+ return nil, err+ }+ if closeErr != nil {+ return nil, closeErr+ }+ targetPath := string(b)++ // Convert to a path relative to the root+ // Symlinks are relative to their file node+ if !path.IsAbs(targetPath) {+ basePath := path.Dir(f.Path())+ targetPath = path.Join(basePath, targetPath)+ }++ // Clean the path, rclone style+ targetPath = path.Clean(targetPath)+ if targetPath == "." {+ targetPath = ""+ }++ // Check if we've already seen this path+ if _, ok := seen[targetPath]; ok {+ return nil, ELOOP+ }+ seen[targetPath] = struct{}{}++ // Resolve the targetPath into a node+ target, err = f.d.vfs.Stat(targetPath)+ if err != nil {+ return nil, err+ }++ // Return node as it must be the destination if not a file+ var ok bool+ f, ok = target.(*File)+ if !ok {+ return target, nil+ }+ }+ return nil, ELOOP+}++// Open also also implements the internal flag o_SYMLINK which instead+// of opening the file a symlink points to, opens the symlink itself.+// This is used for reading and writing the symlink and shouldn't be+// used externally.+const o_SYMLINK = 0x4000_0000 //nolint:revive+// Open a file according to the flags provided//// O_RDONLY open the file read-only.@@ -694,6 +835,16 @@ func (f *File) Open(flags int) (fd Handle, err error) {rdwrMode = flags & accessModeMask)+ // If this is a symlink, then resolve it+ if f.IsSymlink() && flags&o_SYMLINK == 0 {+ target, err := f.resolveNode()+ if err != nil {+ return nil, err+ }+ return target.Open(flags)+ }+ flags &^= o_SYMLINK+// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html// The result of using O_TRUNC with O_RDONLY is undefined.// Linux seems to truncate the file, but we prefer to return EINVAL@@ -738,7 +889,7 @@ func (f *File) Open(flags int) (fd Handle, err error) {d := f.df.mu.RUnlock()CacheMode := d.vfs.Opt.CacheMode- if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.Path()) || d.vfs.cache.Exists(f.Path())) {+ if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.CachePath()) || d.vfs.cache.Exists(f.CachePath())) {fd, err = f.openRW(flags)} else if read && write {if CacheMode >= vfscommon.CacheModeMinimal {diff --git a/vfs/read_write.go b/vfs/read_write.goindex 4130e68b0..56b9c467d 100644--- a/vfs/read_write.go+++ b/vfs/read_write.go@@ -43,7 +43,7 @@ func (fh *RWFileHandle) Unlock() error {func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {defer log.Trace(f.Path(), "")("err=%v", &err)// get an item to represent this from the cache- item := d.vfs.cache.Item(f.Path())+ item := d.vfs.cache.Item(f.CachePath())exists := f.exists() || (item.Exists() && !item.WrittenBack())diff --git a/vfs/vfs.go b/vfs/vfs.goindex 2e60c8fb6..aa6fe15eb 100644--- a/vfs/vfs.go+++ b/vfs/vfs.go@@ -791,6 +791,9 @@ func (vfs *VFS) WriteFile(name string, data []byte, perm os.FileMode) (err error}// AddVirtual adds the object (file or dir) to the directory cache+//+// This is used by the vfs cache to insert objects that are uploading+// into the directory tree.func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {remote = strings.TrimRight(remote, "/")var dir *Dir@@ -808,3 +811,85 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {dir.AddVirtual(leaf, size, false)return nil}++// Readlink returns the destination of the named symbolic link.+// If there is an error, it will be of type *PathError.+func (vfs *VFS) Readlink(name string) (s string, err error) {+ if !vfs.Opt.Links {+ fs.Errorf(nil, "symlinks not supported without the --links flag: %v", name)+ return "", ENOSYS+ }+ node, err := vfs.Stat(name)+ if err != nil {+ return "", err+ }+ file, ok := node.(*File)+ if !ok || !file.IsSymlink() {+ return "", EINVAL // not a symlink+ }+ fd, err := file.Open(os.O_RDONLY | o_SYMLINK)+ if err != nil {+ return "", err+ }+ defer fs.CheckClose(fd, &err)+ b, err := io.ReadAll(fd)+ if err != nil {+ return "", err+ }+ return string(b), nil+}++// CreateSymlink creates newname as a symbolic link to oldname.+// On Windows, a symlink to a non-existent oldname creates a file symlink;+// if oldname is later created as a directory the symlink will not work.+// It returns the node created+func (vfs *VFS) CreateSymlink(oldname, newname string) (Node, error) {+ if !vfs.Opt.Links {+ fs.Errorf(newname, "symlinks not supported without the --links flag")+ return nil, ENOSYS+ }++ // Destination can't exist+ _, err := vfs.Stat(newname)+ if err == nil {+ return nil, EEXIST+ } else if err != ENOENT {+ return nil, err+ }++ // Find the parent+ dir, leaf, err := vfs.StatParent(newname)+ if err != nil {+ return nil, err+ }++ // Create the file node+ flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | o_SYMLINK+ file, err := dir.Create(leaf, flags)+ if err != nil {+ return nil, err+ }++ // Force the file to be a link+ file.setSymlink()++ // Open the file+ fh, err := file.Open(flags)+ if err != nil {+ return nil, err+ }+ defer fs.CheckClose(fh, &err)++ // Write the symlink data+ _, err = fh.Write([]byte(strings.ReplaceAll(oldname, "\\", "/")))++ return file, nil+}++// Symlink creates newname as a symbolic link to oldname.+// On Windows, a symlink to a non-existent oldname creates a file symlink;+// if oldname is later created as a directory the symlink will not work.+func (vfs *VFS) Symlink(oldname, newname string) error {+ _, err := vfs.CreateSymlink(oldname, newname)+ return err+}diff --git a/vfs/write.go b/vfs/write.goindex c799f98d5..8e74b45a8 100644--- a/vfs/write.go+++ b/vfs/write.go@@ -37,6 +37,9 @@ var ()func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHandle, error) {+ if f.IsSymlink() {+ remote += fs.LinkSuffix+ }fh := &WriteFileHandle{remote: remote,flags: flags,