"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: fix open files disappearing from directory listings
Nick Craig-Wood6091a0362be3
diff --git a/vfs/dir.go b/vfs/dir.goindex a76d00cb1..da0f8d47c 100644--- a/vfs/dir.go+++ b/vfs/dir.go@@ -40,7 +40,7 @@ type Dir struct {modTimeMu sync.Mutex // protects the followingmodTime time.Time- _hasVirtual atomic.Bool // shows if the directory has virtual entries+ _virtuals atomic.Int32 // number of virtual directory entries in this directory and children}//go:generate stringer -type=vState@@ -67,7 +67,6 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {items: make(map[string]Node),}d.cleanupTimer = time.AfterFunc(time.Duration(vfs.Opt.DirCacheTime*2), d.cacheCleanup)- d.setHasVirtual(false)return d}@@ -198,58 +197,60 @@ func (d *Dir) Node() Node {return d}-// hasVirtual returns whether the directory has virtual entries+// hasVirtual returns whether the directory or children has virtual entriesfunc (d *Dir) hasVirtual() bool {- return d._hasVirtual.Load()+ return d._virtuals.Load() != 0}-// setHasVirtual sets the hasVirtual flag for the directory-func (d *Dir) setHasVirtual(hasVirtual bool) {- d._hasVirtual.Store(hasVirtual)+// addVirtual adds n virtual items to this directory and all of its parents+func (d *Dir) addVirtual(n int32) {+ for d != nil {+ d._virtuals.Add(n)+ d = d.parent+ }}// ForgetAll forgets directory entries for this directory and any children.//// It does not invalidate or clear the cache of the parent directory.//-// It returns true if the directory or any of its children had virtual entries-// so could not be forgotten. Children which didn't have virtual entries and-// children with virtual entries will be forgotten even if true is returned.+// It returns true if the directory or any of its children had virtual+// entries so could not be forgotten. Children which didn't have+// virtual entries will be forgotten even if true is returned.func (d *Dir) ForgetAll() (hasVirtual bool) {+ // We run this part with RLock only to avoid deadlocks in the recursion+d.mu.RLock()fs.Debugf(d.path, "forgetting directory cache")for _, node := range d.items {if dir, ok := node.(*Dir); ok {- if dir.ForgetAll() {- d.setHasVirtual(true)- }+ dir.ForgetAll()}}d.mu.RUnlock()+ // We run this part with Lock so we can modify the Dir+d.mu.Lock()defer d.mu.Unlock()// Purge any unnecessary virtual entriesd._purgeVirtual()- d.read = time.Time{}-- // Check if this dir has virtual entries- if len(d.virtual) != 0 {- d.setHasVirtual(true)- }-// Don't clear directory entries if there are virtual entries in this// directory or any children- if !d.hasVirtual() {+ hasVirtual = d.hasVirtual()+ if !hasVirtual {+ d.read = time.Time{}d.items = make(map[string]Node)d.cleanupTimer.Stop()+ } else {+ d.cleanupTimer.Reset(time.Duration(d.vfs.Opt.DirCacheTime * 2))}- return d.hasVirtual()+ return hasVirtual}// forgetDirPath clears the cache for itself and all subdirectories if@@ -448,8 +449,10 @@ func (d *Dir) addObject(node Node) {if node.IsDir() {vAdd = vAddDir}+ if _, found := d.virtual[leaf]; !found {+ d.addVirtual(1)+ }d.virtual[leaf] = vAdd- d.setHasVirtual(true)fs.Debugf(d.path, "Added virtual directory entry %v: %q", vAdd, leaf)d.mu.Unlock()}@@ -502,8 +505,10 @@ func (d *Dir) delObject(leaf string) {if d.virtual == nil {d.virtual = make(map[string]vState)}+ if _, found := d.virtual[leaf]; !found {+ d.addVirtual(1)+ }d.virtual[leaf] = vDel- d.setHasVirtual(true)fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)d.mu.Unlock()}@@ -590,9 +595,9 @@ func (d *Dir) _deleteVirtual(name string) {return}delete(d.virtual, name)+ d.addVirtual(-1)if len(d.virtual) == 0 {d.virtual = nil- d.setHasVirtual(false)}fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)}diff --git a/vfs/dir_test.go b/vfs/dir_test.goindex 190ba53d0..555cacf1f 100644--- a/vfs/dir_test.go+++ b/vfs/dir_test.go@@ -607,3 +607,51 @@ func TestDirRename(t *testing.T) {func TestDirStructSize(t *testing.T) {t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{}))}++// Check that open files appear in the directory listing properly after a forget+func TestDirFileOpen(t *testing.T) {+ _, vfs, dir, _ := dirCreate(t)++ assert.False(t, dir.hasVirtual())+ assert.False(t, dir.parent.hasVirtual())++ _, err := dir.Mkdir("sub")+ require.NoError(t, err)++ assert.True(t, dir.hasVirtual())+ assert.True(t, dir.parent.hasVirtual())++ fd0, err := vfs.Create("dir/sub/file0")+ require.NoError(t, err)+ _, err = fd0.Write([]byte("hello"))+ require.NoError(t, err)+ defer func() {+ require.NoError(t, fd0.Close())+ }()++ fd2, err := vfs.Create("dir/sub/file2")+ require.NoError(t, err)+ _, err = fd2.Write([]byte("hello world!"))+ require.NoError(t, err)+ require.NoError(t, fd2.Close())+ assert.True(t, dir.hasVirtual())++ assert.True(t, dir.hasVirtual())+ assert.True(t, dir.parent.hasVirtual())++ // Now forget the directory+ hasVirtual := dir.parent.ForgetAll()+ assert.True(t, hasVirtual)++ assert.True(t, dir.hasVirtual())+ assert.True(t, dir.parent.hasVirtual())++ // Check the files can still be found+ fi, err := vfs.Stat("dir/sub/file0")+ require.NoError(t, err)+ assert.Equal(t, int64(5), fi.Size())++ fi, err = vfs.Stat("dir/sub/file2")+ require.NoError(t, err)+ assert.Equal(t, int64(12), fi.Size())+}