Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

vfs: fix open files disappearing from directory listings

Nick Craig-Wood6091a0362be3
diff --git a/vfs/dir.go b/vfs/dir.go
index a76d00cb1..da0f8d47c 100644
--- a/vfs/dir.go
+++ b/vfs/dir.go
@@ -40,7 +40,7 @@ type Dir struct {
modTimeMu sync.Mutex // protects the following
modTime time.Time
- _hasVirtual atomic.Bool // shows if the directory has virtual entries
+ _virtuals atomic.Int32 // number of virtual directory entries in this directory and children
}
//go:generate stringer -type=vState
@@ -67,7 +67,6 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
items: make(map[string]Node),
}
d.cleanupTimer = time.AfterFunc(time.Duration(vfs.Opt.DirCacheTime*2), d.cacheCleanup)
- d.setHasVirtual(false)
return d
}
@@ -198,58 +197,60 @@ func (d *Dir) Node() Node {
return d
}
-// hasVirtual returns whether the directory has virtual entries
+// hasVirtual returns whether the directory or children has virtual entries
func (d *Dir) hasVirtual() bool {
- return d._hasVirtual.Load()
+ return d._virtuals.Load() != 0
}
-// setHasVirtual sets the hasVirtual flag for the directory
-func (d *Dir) setHasVirtual(hasVirtual bool) {
- d._hasVirtual.Store(hasVirtual)
+// addVirtual adds n virtual items to this directory and all of its parents
+func (d *Dir) addVirtual(n int32) {
+ for d != nil {
+ d._virtuals.Add(n)
+ d = d.parent
+ }
}
// ForgetAll forgets directory entries for this directory and any children.
//
// It does not invalidate or clear the cache of the parent directory.
//
-// It returns true if the directory or any of its children had virtual entries
-// so could not be forgotten. Children which didn't have virtual entries and
-// children with virtual entries will be forgotten even if true is returned.
+// It returns true if the directory or any of its children had virtual
+// entries so could not be forgotten. Children which didn't have
+// virtual entries will be forgotten even if true is returned.
func (d *Dir) ForgetAll() (hasVirtual bool) {
+ // We run this part with RLock only to avoid deadlocks in the recursion
+
d.mu.RLock()
fs.Debugf(d.path, "forgetting directory cache")
for _, node := range d.items {
if dir, ok := node.(*Dir); ok {
- if dir.ForgetAll() {
- d.setHasVirtual(true)
- }
+ dir.ForgetAll()
}
}
d.mu.RUnlock()
+ // We run this part with Lock so we can modify the Dir
+
d.mu.Lock()
defer d.mu.Unlock()
// Purge any unnecessary virtual entries
d._purgeVirtual()
- d.read = time.Time{}
-
- // Check if this dir has virtual entries
- if len(d.virtual) != 0 {
- d.setHasVirtual(true)
- }
-
// Don't clear directory entries if there are virtual entries in this
// directory or any children
- if !d.hasVirtual() {
+ hasVirtual = d.hasVirtual()
+ if !hasVirtual {
+ d.read = time.Time{}
d.items = make(map[string]Node)
d.cleanupTimer.Stop()
+ } else {
+ d.cleanupTimer.Reset(time.Duration(d.vfs.Opt.DirCacheTime * 2))
}
- return d.hasVirtual()
+ return hasVirtual
}
// forgetDirPath clears the cache for itself and all subdirectories if
@@ -448,8 +449,10 @@ func (d *Dir) addObject(node Node) {
if node.IsDir() {
vAdd = vAddDir
}
+ if _, found := d.virtual[leaf]; !found {
+ d.addVirtual(1)
+ }
d.virtual[leaf] = vAdd
- d.setHasVirtual(true)
fs.Debugf(d.path, "Added virtual directory entry %v: %q", vAdd, leaf)
d.mu.Unlock()
}
@@ -502,8 +505,10 @@ func (d *Dir) delObject(leaf string) {
if d.virtual == nil {
d.virtual = make(map[string]vState)
}
+ if _, found := d.virtual[leaf]; !found {
+ d.addVirtual(1)
+ }
d.virtual[leaf] = vDel
- d.setHasVirtual(true)
fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)
d.mu.Unlock()
}
@@ -590,9 +595,9 @@ func (d *Dir) _deleteVirtual(name string) {
return
}
delete(d.virtual, name)
+ d.addVirtual(-1)
if len(d.virtual) == 0 {
d.virtual = nil
- d.setHasVirtual(false)
}
fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
}
diff --git a/vfs/dir_test.go b/vfs/dir_test.go
index 190ba53d0..555cacf1f 100644
--- a/vfs/dir_test.go
+++ b/vfs/dir_test.go
@@ -607,3 +607,51 @@ func TestDirRename(t *testing.T) {
func TestDirStructSize(t *testing.T) {
t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{}))
}
+
+// Check that open files appear in the directory listing properly after a forget
+func TestDirFileOpen(t *testing.T) {
+ _, vfs, dir, _ := dirCreate(t)
+
+ assert.False(t, dir.hasVirtual())
+ assert.False(t, dir.parent.hasVirtual())
+
+ _, err := dir.Mkdir("sub")
+ require.NoError(t, err)
+
+ assert.True(t, dir.hasVirtual())
+ assert.True(t, dir.parent.hasVirtual())
+
+ fd0, err := vfs.Create("dir/sub/file0")
+ require.NoError(t, err)
+ _, err = fd0.Write([]byte("hello"))
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, fd0.Close())
+ }()
+
+ fd2, err := vfs.Create("dir/sub/file2")
+ require.NoError(t, err)
+ _, err = fd2.Write([]byte("hello world!"))
+ require.NoError(t, err)
+ require.NoError(t, fd2.Close())
+ assert.True(t, dir.hasVirtual())
+
+ assert.True(t, dir.hasVirtual())
+ assert.True(t, dir.parent.hasVirtual())
+
+ // Now forget the directory
+ hasVirtual := dir.parent.ForgetAll()
+ assert.True(t, hasVirtual)
+
+ assert.True(t, dir.hasVirtual())
+ assert.True(t, dir.parent.hasVirtual())
+
+ // Check the files can still be found
+ fi, err := vfs.Stat("dir/sub/file0")
+ require.NoError(t, err)
+ assert.Equal(t, int64(5), fi.Size())
+
+ fi, err = vfs.Stat("dir/sub/file2")
+ require.NoError(t, err)
+ assert.Equal(t, int64(12), fi.Size())
+}