Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs/cache: fix parent not getting pinned when remote is a file

nielash9b4b3033da25
diff --git a/fs/cache/cache.go b/fs/cache/cache.go
index ae86ac082..e2c104395 100644
--- a/fs/cache/cache.go
+++ b/fs/cache/cache.go
@@ -4,6 +4,7 @@ package cache
import (
"context"
"runtime"
+ "strings"
"sync"
"github.com/rclone/rclone/fs"
@@ -12,10 +13,11 @@ import (
)
var (
- once sync.Once // creation
- c *cache.Cache
- mu sync.Mutex // mutex to protect remap
- remap = map[string]string{} // map user supplied names to canonical names
+ once sync.Once // creation
+ c *cache.Cache
+ mu sync.Mutex // mutex to protect remap
+ remap = map[string]string{} // map user supplied names to canonical names - [fsString]canonicalName
+ childParentMap = map[string]string{} // tracks a one-to-many relationship between parent dirs and their direct children files - [child]parent
)
// Create the cache just once
@@ -57,6 +59,39 @@ func addMapping(fsString, canonicalName string) {
mu.Unlock()
}
+// addChild tracks known file (child) to directory (parent) relationships.
+// Note that the canonicalName of a child will always equal that of its parent,
+// but not everything with an equal canonicalName is a child.
+// It could be an alias or overridden version of a directory.
+func addChild(child, parent string) {
+ if child == parent {
+ return
+ }
+ mu.Lock()
+ childParentMap[child] = parent
+ mu.Unlock()
+}
+
+// returns true if name is definitely known to be a child (i.e. a file, not a dir).
+// returns false if name is a dir or if we don't know.
+func isChild(child string) bool {
+ mu.Lock()
+ _, found := childParentMap[child]
+ mu.Unlock()
+ return found
+}
+
+// ensures that we return fs.ErrorIsFile when necessary
+func getError(fsString string, err error) error {
+ if err != nil && err != fs.ErrorIsFile {
+ return err
+ }
+ if isChild(fsString) {
+ return fs.ErrorIsFile
+ }
+ return nil
+}
+
// GetFn gets an fs.Fs named fsString either from the cache or creates
// it afresh with the create function
func GetFn(ctx context.Context, fsString string, create func(ctx context.Context, fsString string) (fs.Fs, error)) (f fs.Fs, err error) {
@@ -69,31 +104,39 @@ func GetFn(ctx context.Context, fsString string, create func(ctx context.Context
created = ok
return f, ok, err
})
+ f, ok := value.(fs.Fs)
if err != nil && err != fs.ErrorIsFile {
+ if ok {
+ return f, err // for possible future uses of PutErr
+ }
return nil, err
}
- f = value.(fs.Fs)
// Check we stored the Fs at the canonical name
if created {
canonicalName := fs.ConfigString(f)
if canonicalName != canonicalFsString {
- // Note that if err == fs.ErrorIsFile at this moment
- // then we can't rename the remote as it will have the
- // wrong error status, we need to add a new one.
- if err == nil {
+ if err == nil { // it's a dir
fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", canonicalFsString, canonicalName)
value, found := c.Rename(canonicalFsString, canonicalName)
if found {
f = value.(fs.Fs)
}
addMapping(canonicalFsString, canonicalName)
- } else {
- fs.Debugf(nil, "fs cache: adding new entry for parent of %q, %q", canonicalFsString, canonicalName)
- Put(canonicalName, f)
+ } else { // it's a file
+ // the fs we cache is always the file's parent, never the file,
+ // but we use the childParentMap to return the correct error status based on the fsString passed in.
+ fs.Debugf(nil, "fs cache: renaming child cache item %q to be canonical for parent %q", canonicalFsString, canonicalName)
+ value, found := c.Rename(canonicalFsString, canonicalName) // rename the file entry to parent
+ if found {
+ f = value.(fs.Fs) // if parent already exists, use it
+ }
+ Put(canonicalName, f) // force err == nil for the cache
+ addMapping(canonicalFsString, canonicalName) // note the fsString-canonicalName connection for future lookups
+ addChild(fsString, canonicalName) // note the file-directory connection for future lookups
}
}
}
- return f, err
+ return f, getError(fsString, err) // ensure fs.ErrorIsFile is returned when necessary
}
// Pin f into the cache until Unpin is called
@@ -111,7 +154,6 @@ func PinUntilFinalized(f fs.Fs, x interface{}) {
runtime.SetFinalizer(x, func(_ interface{}) {
Unpin(f)
})
-
}
// Unpin f from the cache
@@ -174,6 +216,9 @@ func PutErr(fsString string, f fs.Fs, err error) {
canonicalName := fs.ConfigString(f)
c.PutErr(canonicalName, f, err)
addMapping(fsString, canonicalName)
+ if err == fs.ErrorIsFile {
+ addChild(fsString, canonicalName)
+ }
}
// Put puts an fs.Fs named fsString into the cache
@@ -186,6 +231,7 @@ func Put(fsString string, f fs.Fs) {
// Returns number of entries deleted
func ClearConfig(name string) (deleted int) {
createOnFirstUse()
+ ClearMappingsPrefix(name)
return c.DeletePrefix(name + ":")
}
@@ -193,6 +239,7 @@ func ClearConfig(name string) (deleted int) {
func Clear() {
createOnFirstUse()
c.Clear()
+ ClearMappings()
}
// Entries returns the number of entries in the cache
@@ -200,3 +247,39 @@ func Entries() int {
createOnFirstUse()
return c.Entries()
}
+
+// ClearMappings removes everything from remap and childParentMap
+func ClearMappings() {
+ mu.Lock()
+ defer mu.Unlock()
+ remap = map[string]string{}
+ childParentMap = map[string]string{}
+}
+
+// ClearMappingsPrefix deletes all mappings to parents with given prefix
+//
+// Returns number of entries deleted
+func ClearMappingsPrefix(prefix string) (deleted int) {
+ mu.Lock()
+ do := func(mapping map[string]string) {
+ for key, val := range mapping {
+ if !strings.HasPrefix(val, prefix) {
+ continue
+ }
+ delete(mapping, key)
+ deleted++
+ }
+ }
+ do(remap)
+ do(childParentMap)
+ mu.Unlock()
+ return deleted
+}
+
+// EntriesWithPinCount returns the number of pinned and unpinned entries in the cache
+//
+// Each entry is counted only once, regardless of entry.pinCount
+func EntriesWithPinCount() (pinned, unpinned int) {
+ createOnFirstUse()
+ return c.EntriesWithPinCount()
+}
diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go
index acddac80c..d9d54fa2e 100644
--- a/fs/cache/cache_test.go
+++ b/fs/cache/cache_test.go
@@ -24,7 +24,7 @@ func mockNewFs(t *testing.T) func(ctx context.Context, path string) (fs.Fs, erro
switch path {
case "mock:/":
return mockfs.NewFs(ctx, "mock", "/", nil)
- case "mock:/file.txt", "mock:file.txt":
+ case "mock:/file.txt", "mock:file.txt", "mock:/file2.txt", "mock:file2.txt":
fMock, err := mockfs.NewFs(ctx, "mock", "/", nil)
require.NoError(t, err)
return fMock, fs.ErrorIsFile
@@ -55,6 +55,7 @@ func TestGet(t *testing.T) {
}
func TestGetFile(t *testing.T) {
+ defer ClearMappings()
create := mockNewFs(t)
assert.Equal(t, 0, Entries())
@@ -63,7 +64,7 @@ func TestGetFile(t *testing.T) {
require.Equal(t, fs.ErrorIsFile, err)
require.NotNil(t, f)
- assert.Equal(t, 2, Entries())
+ assert.Equal(t, 1, Entries())
f2, err := GetFn(context.Background(), "mock:/file.txt", create)
require.Equal(t, fs.ErrorIsFile, err)
@@ -71,7 +72,7 @@ func TestGetFile(t *testing.T) {
assert.Equal(t, f, f2)
- // check parent is there too
+ // check it is also found when referred to by parent name
f2, err = GetFn(context.Background(), "mock:/", create)
require.Nil(t, err)
require.NotNil(t, f2)
@@ -80,6 +81,7 @@ func TestGetFile(t *testing.T) {
}
func TestGetFile2(t *testing.T) {
+ defer ClearMappings()
create := mockNewFs(t)
assert.Equal(t, 0, Entries())
@@ -88,7 +90,7 @@ func TestGetFile2(t *testing.T) {
require.Equal(t, fs.ErrorIsFile, err)
require.NotNil(t, f)
- assert.Equal(t, 2, Entries())
+ assert.Equal(t, 1, Entries())
f2, err := GetFn(context.Background(), "mock:file.txt", create)
require.Equal(t, fs.ErrorIsFile, err)
@@ -96,7 +98,7 @@ func TestGetFile2(t *testing.T) {
assert.Equal(t, f, f2)
- // check parent is there too
+ // check it is also found when referred to by parent name
f2, err = GetFn(context.Background(), "mock:/", create)
require.Nil(t, err)
require.NotNil(t, f2)
@@ -124,22 +126,22 @@ func TestPutErr(t *testing.T) {
assert.Equal(t, 0, Entries())
- PutErr("mock:file.txt", f, fs.ErrorIsFile)
+ PutErr("mock:/", f, fs.ErrorNotFoundInConfigFile)
assert.Equal(t, 1, Entries())
- fNew, err := GetFn(context.Background(), "mock:file.txt", create)
- require.Equal(t, fs.ErrorIsFile, err)
+ fNew, err := GetFn(context.Background(), "mock:/", create)
+ require.Equal(t, fs.ErrorNotFoundInConfigFile, err)
require.Equal(t, f, fNew)
assert.Equal(t, 1, Entries())
// Check canonicalisation
- PutErr("mock:/file.txt", f, fs.ErrorIsFile)
+ PutErr("mock:/file.txt", f, fs.ErrorNotFoundInConfigFile)
fNew, err = GetFn(context.Background(), "mock:/file.txt", create)
- require.Equal(t, fs.ErrorIsFile, err)
+ require.Equal(t, fs.ErrorNotFoundInConfigFile, err)
require.Equal(t, f, fNew)
assert.Equal(t, 1, Entries())
@@ -190,6 +192,75 @@ func TestPin(t *testing.T) {
Unpin(f2)
}
+func TestPinFile(t *testing.T) {
+ defer ClearMappings()
+ create := mockNewFs(t)
+
+ // Test pinning and unpinning nonexistent
+ f, err := mockfs.NewFs(context.Background(), "mock", "/file.txt", nil)
+ require.NoError(t, err)
+ Pin(f)
+ Unpin(f)
+
+ // Now test pinning an existing
+ f2, err := GetFn(context.Background(), "mock:/file.txt", create)
+ require.Equal(t, fs.ErrorIsFile, err)
+ assert.Equal(t, 1, len(childParentMap))
+
+ Pin(f2)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned := EntriesWithPinCount()
+ assert.Equal(t, 1, pinned)
+ assert.Equal(t, 0, unpinned)
+
+ Unpin(f2)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned = EntriesWithPinCount()
+ assert.Equal(t, 0, pinned)
+ assert.Equal(t, 1, unpinned)
+
+ // try a different child of the same parent, and parent
+ // should not add additional cache items
+ called = 0 // this one does create() because we haven't seen it before and don't yet know it's a file
+ f3, err := GetFn(context.Background(), "mock:/file2.txt", create)
+ assert.Equal(t, fs.ErrorIsFile, err)
+ assert.Equal(t, 1, Entries())
+ assert.Equal(t, 2, len(childParentMap))
+
+ parent, err := GetFn(context.Background(), "mock:/", create)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, Entries())
+ assert.Equal(t, 2, len(childParentMap))
+
+ Pin(f3)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned = EntriesWithPinCount()
+ assert.Equal(t, 1, pinned)
+ assert.Equal(t, 0, unpinned)
+
+ Unpin(f3)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned = EntriesWithPinCount()
+ assert.Equal(t, 0, pinned)
+ assert.Equal(t, 1, unpinned)
+
+ Pin(parent)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned = EntriesWithPinCount()
+ assert.Equal(t, 1, pinned)
+ assert.Equal(t, 0, unpinned)
+
+ Unpin(parent)
+ assert.Equal(t, 1, Entries())
+ pinned, unpinned = EntriesWithPinCount()
+ assert.Equal(t, 0, pinned)
+ assert.Equal(t, 1, unpinned)
+
+ // all 3 should have equal configstrings
+ assert.Equal(t, fs.ConfigString(f2), fs.ConfigString(f3))
+ assert.Equal(t, fs.ConfigString(f2), fs.ConfigString(parent))
+}
+
func TestClearConfig(t *testing.T) {
create := mockNewFs(t)
@@ -198,9 +269,9 @@ func TestClearConfig(t *testing.T) {
_, err := GetFn(context.Background(), "mock:/file.txt", create)
require.Equal(t, fs.ErrorIsFile, err)
- assert.Equal(t, 2, Entries()) // file + parent
+ assert.Equal(t, 1, Entries())
- assert.Equal(t, 2, ClearConfig("mock"))
+ assert.Equal(t, 1, ClearConfig("mock"))
assert.Equal(t, 0, Entries())
}
diff --git a/fs/rc/cache_test.go b/fs/rc/cache_test.go
index 0a351cee7..15952fa04 100644
--- a/fs/rc/cache_test.go
+++ b/fs/rc/cache_test.go
@@ -22,7 +22,7 @@ func mockNewFs(t *testing.T) func() {
require.NoError(t, err)
cache.Put("mock:/", f)
cache.Put(":mock:/", f)
- f, err = mockfs.NewFs(ctx, "mock", "dir/file.txt", nil)
+ f, err = mockfs.NewFs(ctx, "mock", "dir/", nil)
require.NoError(t, err)
cache.PutErr("mock:dir/file.txt", f, fs.ErrorIsFile)
return func() {
diff --git a/lib/cache/cache.go b/lib/cache/cache.go
index fc73eab68..6f841d182 100644
--- a/lib/cache/cache.go
+++ b/lib/cache/cache.go
@@ -260,3 +260,19 @@ func (c *Cache) SetFinalizer(finalize func(interface{})) {
c.finalize = finalize
c.mu.Unlock()
}
+
+// EntriesWithPinCount returns the number of pinned and unpinned entries in the cache
+//
+// Each entry is counted only once, regardless of entry.pinCount
+func (c *Cache) EntriesWithPinCount() (pinned, unpinned int) {
+ c.mu.Lock()
+ for _, entry := range c.cache {
+ if entry.pinCount <= 0 {
+ unpinned++
+ } else {
+ pinned++
+ }
+ }
+ c.mu.Unlock()
+ return pinned, unpinned
+}