"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: fix crash when multiple mounts or servers share the same VFS
Nick Craig-Woode006d7c13fed
diff --git a/cmd/mount/dir.go b/cmd/mount/dir.goindex ad657bc19..272ce1329 100644--- a/cmd/mount/dir.go+++ b/cmd/mount/dir.go@@ -81,7 +81,7 @@ func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.Loresp.EntryValid = time.Duration(d.fsys.opt.AttrTimeout)// Check the mnode to see if it has a fuse Node cached// We must return the same fuse nodes for vfs Nodes- node, ok := mnode.Sys().(fusefs.Node)+ node, ok := mnode.Aux(d.fsys).(fusefs.Node)if ok {return node, nil}@@ -94,7 +94,7 @@ func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.Lopanic("bad type")}// Cache the node for later- mnode.SetSys(node)+ mnode.SetAux(d.fsys, node)return node, nil}@@ -158,7 +158,7 @@ func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.Crreturn nil, nil, translateError(err)}node = &File{file, d.fsys}- file.SetSys(node) // cache the FUSE node for later+ file.SetAux(d.fsys, node) // cache the FUSE node for laterreturn node, &FileHandle{fh}, err}@@ -172,7 +172,7 @@ func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (node fusefs.Noreturn nil, translateError(err)}node = &Dir{dir, d.fsys}- dir.SetSys(node) // cache the FUSE node for later+ dir.SetAux(d.fsys, node) // cache the FUSE node for laterreturn node, nil}diff --git a/cmd/mount2/node.go b/cmd/mount2/node.goindex 705a14f38..3b60c651f 100644--- a/cmd/mount2/node.go+++ b/cmd/mount2/node.go@@ -30,7 +30,7 @@ var _ fusefs.InodeEmbedder = (*Node)(nil)func newNode(fsys *FS, vfsNode vfs.Node) (node *Node) {// Check the vfsNode to see if it has a fuse Node cached// We must return the same fuse nodes for vfs Nodes- node, ok := vfsNode.Sys().(*Node)+ node, ok := vfsNode.Aux(fsys).(*Node)if ok {return node}@@ -39,7 +39,7 @@ func newNode(fsys *FS, vfsNode vfs.Node) (node *Node) {fsys: fsys,}// Cache the node for later- vfsNode.SetSys(node)+ vfsNode.SetAux(fsys, node)return node}diff --git a/vfs/aux.go b/vfs/aux.gonew file mode 100644index 000000000..c6a404266--- /dev/null+++ b/vfs/aux.go@@ -0,0 +1,85 @@+package vfs++import "sync/atomic"++// auxEntry associates an owner with a value attached to a node.+type auxEntry struct {+ owner, value any+}++// aux holds auxiliary values attached to a node, keyed by owner.+//+// It is embedded in Dir and File to provide the Aux, SetAux, Sys and+// SetSys methods of the Node interface.+//+// Reads are lock free. Writes copy the entry list, which is assumed+// to be very short.+type aux struct {+ entries atomic.Pointer[[]auxEntry]+}++// sysOwner is the owner Sys and SetSys attach their value under.+type sysOwner struct{}++// Aux returns the value attached to the node for owner, or nil if+// none is attached.+func (a *aux) Aux(owner any) any {+ if entries := a.entries.Load(); entries != nil {+ for _, entry := range *entries {+ if entry.owner == owner {+ return entry.value+ }+ }+ }+ return nil+}++// SetAux attaches value to the node for owner, replacing any value+// owner attached before. Attaching nil removes owner's value.+//+// Values attached by different owners are independent, so multiple+// users of a shared VFS do not conflict. Owner must be comparable and+// unique to the user - a pointer to the user's filesystem struct is a+// good choice.+func (a *aux) SetAux(owner, value any) {+ for {+ old := a.entries.Load()+ var entries []auxEntry+ if old != nil {+ entries = make([]auxEntry, 0, len(*old)+1)+ for _, entry := range *old {+ if entry.owner != owner {+ entries = append(entries, entry)+ }+ }+ }+ if value != nil {+ entries = append(entries, auxEntry{owner: owner, value: value})+ }+ var next *[]auxEntry+ if len(entries) != 0 {+ next = &entries+ }+ if a.entries.CompareAndSwap(old, next) {+ return+ }+ }+}++// Sys returns the value set with SetSys (can be nil) - satisfies+// os.FileInfo.+//+// This is what callers reading the node through os.FileInfo (e.g. the+// NFS server library) will see. It is reserved for users which need+// to control that; users caching their own data on the node should+// use Aux and SetAux instead. As the VFS may be shared, the value+// stored here should be derived from the node alone so that all users+// store the same value.+func (a *aux) Sys() any {+ return a.Aux(sysOwner{})+}++// SetSys sets the value returned by Sys - see Sys for the contract.+func (a *aux) SetSys(x any) {+ a.SetAux(sysOwner{}, x)+}diff --git a/vfs/aux_test.go b/vfs/aux_test.gonew file mode 100644index 000000000..d0c74c92f--- /dev/null+++ b/vfs/aux_test.go@@ -0,0 +1,73 @@+package vfs++import (+ "fmt"+ "sync"+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestAux(t *testing.T) {+ var a aux+ owner1, owner2 := new(int), new(int)++ // Nothing attached yet+ assert.Nil(t, a.Aux(owner1))+ assert.Nil(t, a.Sys())++ // Values attached by different owners are independent even if+ // they have different types+ a.SetAux(owner1, "potato")+ a.SetAux(owner2, 2)+ assert.Equal(t, "potato", a.Aux(owner1))+ assert.Equal(t, 2, a.Aux(owner2))++ // Replace a value+ a.SetAux(owner1, "sausage")+ assert.Equal(t, "sausage", a.Aux(owner1))++ // Remove a value+ a.SetAux(owner1, nil)+ assert.Nil(t, a.Aux(owner1))+ assert.Equal(t, 2, a.Aux(owner2))++ // Sys is independent of the other owners+ assert.Nil(t, a.Sys())+ a.SetSys(42)+ assert.Equal(t, 42, a.Sys())+ assert.Equal(t, 2, a.Aux(owner2))++ // Changing the type of the value stored must not panic+ a.SetSys("42")+ assert.Equal(t, "42", a.Sys())++ // Remove the remaining values+ a.SetSys(nil)+ a.SetAux(owner2, nil)+ assert.Nil(t, a.entries.Load())+}++func TestAuxConcurrent(t *testing.T) {+ const (+ owners = 4+ iterations = 100+ )+ var (+ a aux+ wg sync.WaitGroup+ )+ for i := range owners {+ wg.Add(1)+ go func() {+ defer wg.Done()+ owner := &i+ for j := range iterations {+ value := fmt.Sprintf("%d-%d", i, j)+ a.SetAux(owner, value)+ assert.Equal(t, value, a.Aux(owner))+ }+ }()+ }+ wg.Wait()+}diff --git a/vfs/dir.go b/vfs/dir.goindex 05fefd930..0d24195bd 100644--- a/vfs/dir.go+++ b/vfs/dir.go@@ -24,6 +24,7 @@ import (// Dir represents a directory entrytype Dir struct {+ aux // values attached by users of the VFSvfs *VFS // read onlyinode uint64 // read only: inode numberf fs.Fs // read only@@ -36,7 +37,6 @@ type Dir struct {read time.Time // time directory entry last readitems map[string]Node // directory entries - can be empty but not nilvirtual map[string]vState // virtual directory entries - may be nil- sys atomic.Value // user defined info to be attached heremodTimeMu sync.Mutex // protects the followingmodTime time.Time@@ -181,16 +181,6 @@ func (d *Dir) Path() (name string) {return d.path}-// Sys returns underlying data source (can be nil) - satisfies Node interface-func (d *Dir) Sys() any {- return d.sys.Load()-}--// SetSys sets the underlying data source (can be nil) - satisfies Node interface-func (d *Dir) SetSys(x any) {- d.sys.Store(x)-}-// Inode returns the inode number - satisfies Node interfacefunc (d *Dir) Inode() uint64 {return d.inodediff --git a/vfs/file.go b/vfs/file.goindex 45909ab11..69a7afcd1 100644--- a/vfs/file.go+++ b/vfs/file.go@@ -40,6 +40,7 @@ import (// File represents a file or a symlinktype File struct {+ aux // values attached by users of the VFSinode uint64 // inode number - read onlysize atomic.Int64 // size of filectx context.Context // context for VFS operations - read only@@ -55,7 +56,6 @@ type File struct {virtualModTime *time.Time // modtime for backends with Precision == fs.ModTimeNotSupportedpendingModTime time.Time // will be applied once o becomes available, i.e. after file was writtenpendingRenameFun func(ctx context.Context) error // will be run/renamed after all writers close- sys atomic.Value // user defined info to be attached herenwriters atomic.Int32 // len(writers)appendMode bool // file was opened with O_APPENDisLink bool // file represents a symlink@@ -184,16 +184,6 @@ func (f *File) CachePath() string {return f._cachePath()}-// Sys returns underlying data source (can be nil) - satisfies Node interface-func (f *File) Sys() any {- return f.sys.Load()-}--// SetSys sets the underlying data source (can be nil) - satisfies Node interface-func (f *File) SetSys(x any) {- f.sys.Store(x)-}-// Inode returns the inode number - satisfies Node interfacefunc (f *File) Inode() uint64 {return f.inodediff --git a/vfs/vfs.go b/vfs/vfs.goindex 9006b798c..aaf94d9bf 100644--- a/vfs/vfs.go+++ b/vfs/vfs.go@@ -69,6 +69,8 @@ type Node interface {Truncate(size int64) errorPath() stringSetSys(any)+ Aux(owner any) any+ SetAux(owner, value any)}// Check interfaces