Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve: fix --auth-proxy stopping VFS cache uploads after 5 minutes

Nick Craig-Wood0caf24af5058
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
index ed0bb3395..d3d7fb7af 100644
--- a/cmd/serve/proxy/proxy.go
+++ b/cmd/serve/proxy/proxy.go
@@ -233,6 +233,13 @@ func New(ctx context.Context, opt *Options, vfsOpt *vfscommon.Options) *Proxy {
entry.vfs.Shutdown()
}
})
+ // Shutting down a VFS stops it writing, so keep one which still
+ // has data to write, e.g. files in the VFS cache waiting to be
+ // uploaded after the transfer which wrote them has finished.
+ p.vfsCache.SetCanExpire(func(value any) bool {
+ entry, ok := value.(cacheEntry)
+ return !ok || entry.vfs == nil || !entry.vfs.Busy()
+ })
return p
}
diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.go
index 411288773..00c99d320 100644
--- a/cmd/serve/proxy/proxy_test.go
+++ b/cmd/serve/proxy/proxy_test.go
@@ -475,3 +475,29 @@ func TestCallAccessKey(t *testing.T) {
_, _, err = p.CallAccessKey("AKID", remoteAddr, false)
require.ErrorContains(t, err, "revoked")
}
+
+// TestBusyVFSNotExpired checks a VFS which still has data to write
+// isn't expired from the cache, and so shut down, until it is written.
+func TestBusyVFSNotExpired(t *testing.T) {
+ opt := Opt
+ opt.AuthProxy = "go run ../servetest/proxy_code.go " + t.TempDir()
+ p := New(context.Background(), &opt, &vfscommon.Opt)
+ defer p.Shutdown()
+ p.vfsCache.SetExpireDuration(time.Millisecond).SetExpireInterval(10 * time.Millisecond)
+
+ VFS, _, err := p.Call("user", "pass", false, "192.0.2.1:1024")
+ require.NoError(t, err)
+ fh, err := VFS.Create("file.txt")
+ require.NoError(t, err)
+
+ assert.Never(t, func() bool {
+ return p.vfsCache.Entries() == 0
+ }, 250*time.Millisecond, 10*time.Millisecond, "VFS expired with a file open for write")
+ assert.Equal(t, int32(1), VFS.Stats()["inUse"], "VFS shut down with a file open for write")
+
+ require.NoError(t, fh.Close())
+ assert.Eventually(t, func() bool {
+ return p.vfsCache.Entries() == 0
+ }, 10*time.Second, 10*time.Millisecond, "VFS not expired once it had nothing to write")
+ assert.Equal(t, int32(0), VFS.Stats()["inUse"], "VFS not shut down once it had nothing to write")
+}