"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve: fix --auth-proxy stopping VFS cache uploads after 5 minutes
Nick Craig-Wood0caf24af5058
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.goindex ed0bb3395..d3d7fb7af 100644--- a/cmd/serve/proxy/proxy.go+++ b/cmd/serve/proxy/proxy.go@@ -233,6 +233,13 @@ func New(ctx context.Context, opt *Options, vfsOpt *vfscommon.Options) *Proxy {entry.vfs.Shutdown()}})+ // Shutting down a VFS stops it writing, so keep one which still+ // has data to write, e.g. files in the VFS cache waiting to be+ // uploaded after the transfer which wrote them has finished.+ p.vfsCache.SetCanExpire(func(value any) bool {+ entry, ok := value.(cacheEntry)+ return !ok || entry.vfs == nil || !entry.vfs.Busy()+ })return p}diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.goindex 411288773..00c99d320 100644--- a/cmd/serve/proxy/proxy_test.go+++ b/cmd/serve/proxy/proxy_test.go@@ -475,3 +475,29 @@ func TestCallAccessKey(t *testing.T) {_, _, err = p.CallAccessKey("AKID", remoteAddr, false)require.ErrorContains(t, err, "revoked")}++// TestBusyVFSNotExpired checks a VFS which still has data to write+// isn't expired from the cache, and so shut down, until it is written.+func TestBusyVFSNotExpired(t *testing.T) {+ opt := Opt+ opt.AuthProxy = "go run ../servetest/proxy_code.go " + t.TempDir()+ p := New(context.Background(), &opt, &vfscommon.Opt)+ defer p.Shutdown()+ p.vfsCache.SetExpireDuration(time.Millisecond).SetExpireInterval(10 * time.Millisecond)++ VFS, _, err := p.Call("user", "pass", false, "192.0.2.1:1024")+ require.NoError(t, err)+ fh, err := VFS.Create("file.txt")+ require.NoError(t, err)++ assert.Never(t, func() bool {+ return p.vfsCache.Entries() == 0+ }, 250*time.Millisecond, 10*time.Millisecond, "VFS expired with a file open for write")+ assert.Equal(t, int32(1), VFS.Stats()["inUse"], "VFS shut down with a file open for write")++ require.NoError(t, fh.Close())+ assert.Eventually(t, func() bool {+ return p.vfsCache.Entries() == 0+ }, 10*time.Second, 10*time.Millisecond, "VFS not expired once it had nothing to write")+ assert.Equal(t, int32(0), VFS.Stats()["inUse"], "VFS not shut down once it had nothing to write")+}